Update eval_notebook.ipynb

eval/eval_notebook.ipynb

@@ -2,18 +2,9 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/Users/romainfayoux/Documents/Programmation/Final_Assignment_Template/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
-      "  from .autonotebook import tqdm as notebook_tqdm\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "import pandas as pd\n",
     "import json\n",
@@ -25,7 +16,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -35,7 +26,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -45,7 +36,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {
@@ -69,18 +60,309 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
-     "ename": "NameError",
-     "evalue": "name 'exact_match_eval' is not defined",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-      "\u001b[31mNameError\u001b[39m                                 Traceback (most recent call last)",
-      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[5]\u001b[39m\u001b[32m, line 8\u001b[39m\n\u001b[32m      6\u001b[39m conciseness_evaluator = bind_evaluator(evaluator=conciseness_evaluator, input_mapping={ \u001b[33m\"\u001b[39m\u001b[33moutput\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33mattributes.output.value\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mexpected\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33mFinal answer\u001b[39m\u001b[33m\"\u001b[39m})\n\u001b[32m      7\u001b[39m question_scorer_eval = bind_evaluator(evaluator=question_scorer, input_mapping={ \u001b[33m\"\u001b[39m\u001b[33moutput\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33mattributes.output.value\u001b[39m\u001b[33m\"\u001b[39m, \u001b[33m\"\u001b[39m\u001b[33mexpected\u001b[39m\u001b[33m\"\u001b[39m: \u001b[33m\"\u001b[39m\u001b[33mFinal answer\u001b[39m\u001b[33m\"\u001b[39m})\n\u001b[32m----> \u001b[39m\u001b[32m8\u001b[39m results_df = \u001b[38;5;28;01mawait\u001b[39;00m async_evaluate_dataframe(agents_merged_df, evaluators=[\u001b[43mexact_match_eval\u001b[49m, conciseness_evaluator, question_scorer_eval])\n",
-      "\u001b[31mNameError\u001b[39m: name 'exact_match_eval' is not defined"
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Evaluating <code>\n",
+      "page_content_log = visit_webpage(url=\"https://en.wikipedia.org/wiki/Wikipedia:Featured_article_candidates/Featured_log/November_2016\")\n",
+      "print(page_content_log)\n",
+      "</code>\n",
+      "Calling tools:\n",
+      "[{'id': 'call_8', 'type': 'function', 'function': {'name': 'python_interpreter', 'arguments': 'page_content_log = visit_webpage(url=\"https://en.wikipedia.org/wiki/Wikipedia:Featured_article_candidates/Featured_log/November_2016\")\\nprint(page_content_log)'}}] as a string.\n",
+      "Evaluating right as a string.\n",
+      "Evaluating The provided problem requires visual inspection of a chess board image which I cannot access in the current environment. To properly solve this, one would need to analyze the specific piece positions in the image to identify a forcing tactical sequence leading to a guaranteed win for Black. Standard approaches would involve identifying forced captures, mating patterns, or immediate tactical threats that Black can execute on their turn. as a string.\n",
+      "Evaluating right as a string.\n",
+      "Evaluating Looking at the Wikipedia information I retrieved earlier, I need to systematically identify studio albums published between 2000 and 2009.\n",
+      "\n",
+      "From the information available:\n",
+      "- Mercedes Sosa won Latin Grammy Awards for Best Folk Album in 2000 (Misa Criolla), 2003 (Acústico), 2006 (Corazón Libre), 2009 (Cantora 1), and 2011 (Deja La Vida Volar)\n",
+      "- These are all studio albums based on her achievements and the nature of Latin Grammy awards for studio albums\n",
+      "- The years 2000, 2003, 2006, and 2009 all fall within our target range of 2000-2009\n",
+      "\n",
+      "Let me verify these are indeed studio albums by considering the patterns in her discography and the information provided.\n",
+      "\n",
+      "```python\n",
+      "# Based on the Wikipedia information, identifying studio albums in the specified period\n",
+      "albums_in_range = [\n",
+      "    {\"name\": \"Misa Criolla\", \"year\": 2000},\n",
+      "    {\"name\": \"Acústico\", \"year\": 2003}, \n",
+      "    {\"name\": \"Corazón Libre\", \"year\": 2006},\n",
+      "    {\"name\": \"Cantora 1\", \"year\": 2009}\n",
+      "]\n",
+      "\n",
+      "# Count studio albums from 2000-2009\n",
+      "count = len([album for album in albums_in_range if 2000 <= album[\"year\"] <= 2009])\n",
+      "print(count)\n",
+      "```\n",
+      "Based on my analysis of Mercedes Sosa's discography from the Wikipedia information, I have identified 4 studio albums that were published between 2000 and 2009 (inclusive):\n",
+      "\n",
+      "1. Misa Criolla (2000)\n",
+      "2. Acústico (2003)\n",
+      "3. Corazón Libre (2006)\n",
+      "4. Cantora 1 (2009)\n",
+      "\n",
+      "These albums are confirmed by the fact that Mercedes Sosa won Latin Grammy Awards for Best Folk Album for each of these releases in those respective years, which indicates they were studio albums. All four fall within the requested time period of 2000-2009.\n",
+      "\n",
+      "Therefore, the answer is 4 studio albums. as a number.Evaluating cornstarch, lemon juice, ripe strawberries, sugar, vanilla extract as a comma separated list.\n",
+      "\n",
+      "String Looking at the Wikipedia information I retrieved earlier I need to systematically identify studio albums published between 2000 and 2009.\n",
+      "\n",
+      "From the information available:\n",
+      "- Mercedes Sosa won Latin Grammy Awards for Best Folk Album in 2000 (Misa Criolla) 2003 (Acústico) 2006 (Corazón Libre) 2009 (Cantora 1) and 2011 (Deja La Vida Volar)\n",
+      "- These are all studio albums based on her achievements and the nature of Latin Grammy awards for studio albums\n",
+      "- The years 2000 2003 2006 and 2009 all fall within our target range of 2000-2009\n",
+      "\n",
+      "Let me verify these are indeed studio albums by considering the patterns in her discography and the information provided.\n",
+      "\n",
+      "```python\n",
+      "# Based on the Wikipedia information identifying studio albums in the specified period\n",
+      "albums_in_range = [\n",
+      "    {\"name\": \"Misa Criolla\" \"year\": 2000}\n",
+      "    {\"name\": \"Acústico\" \"year\": 2003} \n",
+      "    {\"name\": \"Corazón Libre\" \"year\": 2006}\n",
+      "    {\"name\": \"Cantora 1\" \"year\": 2009}\n",
+      "]\n",
+      "\n",
+      "# Count studio albums from 2000-2009\n",
+      "count = len([album for album in albums_in_range if 2000 <= album[\"year\"] <= 2009])\n",
+      "print(count)\n",
+      "```\n",
+      "Based on my analysis of Mercedes Sosa's discography from the Wikipedia information I have identified 4 studio albums that were published between 2000 and 2009 (inclusive):\n",
+      "\n",
+      "1. Misa Criolla (2000)\n",
+      "2. Acústico (2003)\n",
+      "3. Corazón Libre (2006)\n",
+      "4. Cantora 1 (2009)\n",
+      "\n",
+      "These albums are confirmed by the fact that Mercedes Sosa won Latin Grammy Awards for Best Folk Album for each of these releases in those respective years which indicates they were studio albums. All four fall within the requested time period of 2000-2009.\n",
+      "\n",
+      "Therefore the answer is 4 studio albums. cannot be normalized to number str.\n",
+      "Evaluating broccoli, celery, fresh basil, green beans, lettuce, sweet potatoes, zucchini as a comma separated list.\n",
+      "Evaluating Information not available as a string.\n",
+      "Evaluating b,e as a comma separated list.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/Users/romainfayoux/Documents/Programmation/Final_Assignment_Template/eval/scorer.py:61: UserWarning: Answer lists have different lengths, returning False.\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Evaluating Given the issues accessing the specific Wikipedia page directly, I will use an alternative approach to find the information. I'll search for the specific Featured Article about a dinosaur promoted in November 2016 and then look for its nomination details.\n",
+      "\n",
+      "Let's start by searching for the specific Featured Article about a dinosaur promoted in November 2016.\n",
+      "\n",
+      "<code>\n",
+      "# Perform a web search to find the specific Featured Article about a dinosaur promoted in November 2016\n",
+      "search_results = web_search(query=\"Featured Article dinosaur promoted November 2016\")\n",
+      "print(search_results)\n",
+      "</code>\n",
+      "Calling tools:\n",
+      "[{'id': 'call_8', 'type': 'function', 'function': {'name': 'python_interpreter', 'arguments': '# Perform a web search to find the specific Featured Article about a dinosaur promoted in November 2016\\nsearch_results = web_search(query=\"Featured Article dinosaur promoted November 2016\")\\nprint(search_results)'}}] as a string.\n",
+      "Evaluating d5 as a string.\n",
+      "Evaluating right as a string.\n",
+      "Evaluating Given the issues with extracting the discography section using regex, I will manually identify the studio albums released by Mercedes Sosa between 2000 and 2009 based on the information provided in the Wikipedia page.\n",
+      "\n",
+      "From the Wikipedia page, the studio albums section lists the following albums with their release years:\n",
+      "\n",
+      "- Misa Criolla (2000)\n",
+      "- Acústico (2003)\n",
+      "- Corazón Libre (2006)\n",
+      "- Cantora 1 (2009)\n",
+      "\n",
+      "These are the studio albums released by Mercedes Sosa between 2000 and 2009. Therefore, the number of studio albums published by Mercedes Sosa between 2000 and 2009 is 4.\n",
+      "\n",
+      "Final answer: Mercedes Sosa published 4 studio albums between 2000 and 2009. as a number.\n",
+      "String Given the issues with extracting the discography section using regex I will manually identify the studio albums released by Mercedes Sosa between 2000 and 2009 based on the information provided in the Wikipedia page.\n",
+      "\n",
+      "From the Wikipedia page the studio albums section lists the following albums with their release years:\n",
+      "\n",
+      "- Misa Criolla (2000)\n",
+      "- Acústico (2003)\n",
+      "- Corazón Libre (2006)\n",
+      "- Cantora 1 (2009)\n",
+      "\n",
+      "These are the studio albums released by Mercedes Sosa between 2000 and 2009. Therefore the number of studio albums published by Mercedes Sosa between 2000 and 2009 is 4.\n",
+      "\n",
+      "Final answer: Mercedes Sosa published 4 studio albums between 2000 and 2009. cannot be normalized to number str.\n",
+      "Evaluating right as a string.Evaluating Given the issues with parsing the Wikipedia page using regular expressions, I will manually identify the studio albums released by Mercedes Sosa between 2000 and 2009 based on the information provided in the Wikipedia content.\n",
+      "\n",
+      "From the discography section of the Wikipedia page, I can identify the following studio albums and their release years:\n",
+      "\n",
+      "- **Misa Criolla** (2000)\n",
+      "- **Acústico** (2003)\n",
+      "- **Corazón Libre** (2006)\n",
+      "- **Cantora 1** (2009)\n",
+      "\n",
+      "These are the studio albums released by Mercedes Sosa between 2000 and 2009. Therefore, the number of studio albums published by Mercedes Sosa between 2000 and 2009 is **4**.\n",
+      "\n",
+      "Final answer: Mercedes Sosa published 4 studio albums between 2000 and 2009. as a number.\n",
+      "String Given the issues with parsing the Wikipedia page using regular expressions I will manually identify the studio albums released by Mercedes Sosa between 2000 and 2009 based on the information provided in the Wikipedia content.\n",
+      "\n",
+      "From the discography section of the Wikipedia page I can identify the following studio albums and their release years:\n",
+      "\n",
+      "- **Misa Criolla** (2000)\n",
+      "- **Acústico** (2003)\n",
+      "- **Corazón Libre** (2006)\n",
+      "- **Cantora 1** (2009)\n",
+      "\n",
+      "These are the studio albums released by Mercedes Sosa between 2000 and 2009. Therefore the number of studio albums published by Mercedes Sosa between 2000 and 2009 is **4**.\n",
+      "\n",
+      "Final answer: Mercedes Sosa published 4 studio albums between 2000 and 2009. cannot be normalized to number str.\n",
+      "\n",
+      "Evaluating Based on the information gathered from the search results and the analysis of the bird species mentioned, the highest number of bird species on camera simultaneously in the video \"Penguin Chicks Stand Up To Giant Petrel...With The Help of a Friend!\" is **3**. These species are:\n",
+      "\n",
+      "1. Emperor penguin chicks\n",
+      "2. Adélie penguin\n",
+      "3. Giant petrel\n",
+      "\n",
+      "Therefore, the answer to the user's task is that the highest number of bird species to be on camera simultaneously is **3**. as a number.\n",
+      "String Based on the information gathered from the search results and the analysis of the bird species mentioned the highest number of bird species on camera simultaneously in the video \"Penguin Chicks Stand Up To Giant Petrel...With The Help of a Friend!\" is **3**. These species are:\n",
+      "\n",
+      "1. Emperor penguin chicks\n",
+      "2. Adélie penguin\n",
+      "3. Giant petrel\n",
+      "\n",
+      "Therefore the answer to the user's task is that the highest number of bird species to be on camera simultaneously is **3**. cannot be normalized to number str.\n",
+      "Evaluating It seems that the structure of the Wikipedia page is not matching the expected format, causing the regex search to fail. Let's take a different approach by manually inspecting the discography section of the Wikipedia page and extracting the relevant information.\n",
+      "\n",
+      "Based on the content provided in the previous observation, I will manually identify the studio albums released between 2000 and 2009:\n",
+      "\n",
+      "- **2000**: \"Misa Criolla\"\n",
+      "- **2003**: \"Acústico\"\n",
+      "- **2004**: \"Corazón Libre\"\n",
+      "- **2006**: \"Cantora 1\"\n",
+      "- **2009**: \"Deja La Vida Volar\"\n",
+      "\n",
+      "From this list, we can see that Mercedes Sosa released 5 studio albums between 2000 and 2009.\n",
+      "\n",
+      "Final answer: Mercedes Sosa published 5 studio albums between 2000 and 2009. as a number.\n",
+      "String It seems that the structure of the Wikipedia page is not matching the expected format causing the regex search to fail. Let's take a different approach by manually inspecting the discography section of the Wikipedia page and extracting the relevant information.\n",
+      "\n",
+      "Based on the content provided in the previous observation I will manually identify the studio albums released between 2000 and 2009:\n",
+      "\n",
+      "- **2000**: \"Misa Criolla\"\n",
+      "- **2003**: \"Acústico\"\n",
+      "- **2004**: \"Corazón Libre\"\n",
+      "- **2006**: \"Cantora 1\"\n",
+      "- **2009**: \"Deja La Vida Volar\"\n",
+      "\n",
+      "From this list we can see that Mercedes Sosa released 5 studio albums between 2000 and 2009.\n",
+      "\n",
+      "Final answer: Mercedes Sosa published 5 studio albums between 2000 and 2009. cannot be normalized to number str.\n",
+      "Evaluating Given the limitations in accessing the video content directly, I will rely on the video description and comments to provide an answer. Let's manually inspect the video description and comments for any information about the number of bird species on camera simultaneously.\n",
+      "\n",
+      "### Video Description:\n",
+      "The video title is \"Penguin Chicks Stand Up To Giant Petrel...With The Help of a Friend!\" The description mentions penguin chicks and a giant petrel, but it does not specify the number of bird species on camera simultaneously.\n",
+      "\n",
+      "### Comments:\n",
+      "I will now look through the comments to see if any viewers have mentioned the number of bird species on camera simultaneously.\n",
+      "\n",
+      "<code>\n",
+      "url = \"https://www.youtube.com/watch?v=L1vXCYZAYYM\"\n",
+      "comments = get_video_comments(url)\n",
+      "print(comments)\n",
+      "</code>\n",
+      "Calling tools:\n",
+      "[{'id': 'call_8', 'type': 'function', 'function': {'name': 'python_interpreter', 'arguments': 'url = \"https://www.youtube.com/watch?v=L1vXCYZAYYM\"\\ncomments = get_video_comments(url)\\nprint(comments)'}}] as a number.\n",
+      "String Given the limitations in accessing the video content directly I will rely on the video description and comments to provide an answer. Let's manually inspect the video description and comments for any information about the number of bird species on camera simultaneously.\n",
+      "\n",
+      "### Video Description:\n",
+      "The video title is \"Penguin Chicks Stand Up To Giant Petrel...With The Help of a Friend!\" The description mentions penguin chicks and a giant petrel but it does not specify the number of bird species on camera simultaneously.\n",
+      "\n",
+      "### Comments:\n",
+      "I will now look through the comments to see if any viewers have mentioned the number of bird species on camera simultaneously.\n",
+      "\n",
+      "<code>\n",
+      "url = \"https://www.youtube.com/watch?v=L1vXCYZAYYM\"\n",
+      "comments = get_video_comments(url)\n",
+      "print(comments)\n",
+      "</code>\n",
+      "Calling tools:\n",
+      "[{'id': 'call_8' 'type': 'function' 'function': {'name': 'python_interpreter' 'arguments': 'url = \"https://www.youtube.com/watch?v=L1vXCYZAYYM\"\\ncomments = get_video_comments(url)\\nprint(comments)'}}] cannot be normalized to number str.\n",
+      "Evaluating Given the difficulties in parsing the Wikipedia page directly, I will manually extract the relevant information from the discography section of the Mercedes Sosa Wikipedia page.\n",
+      "\n",
+      "Here is the discography section from the Wikipedia page, focusing on the studio albums:\n",
+      "\n",
+      "### Studio albums\n",
+      "- Misa Criolla (2000)\n",
+      "- Acústico (2003)\n",
+      "- Corazón Libre (2006)\n",
+      "- Cantora 1 (2009)\n",
+      "\n",
+      "Based on this information, we can see that Mercedes Sosa released four studio albums between 2000 and 2009 (inclusive).\n",
+      "\n",
+      "Therefore, the final answer is:\n",
+      "**Mercedes Sosa published 4 studio albums between 2000 and 2009.** as a number.\n",
+      "String Given the difficulties in parsing the Wikipedia page directly I will manually extract the relevant information from the discography section of the Mercedes Sosa Wikipedia page.\n",
+      "\n",
+      "Here is the discography section from the Wikipedia page focusing on the studio albums:\n",
+      "\n",
+      "### Studio albums\n",
+      "- Misa Criolla (2000)\n",
+      "- Acústico (2003)\n",
+      "- Corazón Libre (2006)\n",
+      "- Cantora 1 (2009)\n",
+      "\n",
+      "Based on this information we can see that Mercedes Sosa released four studio albums between 2000 and 2009 (inclusive).\n",
+      "\n",
+      "Therefore the final answer is:\n",
+      "**Mercedes Sosa published 4 studio albums between 2000 and 2009.** cannot be normalized to number str.\n",
+      "Evaluating FunkMonk as a string.\n",
+      "Evaluating right as a string.\n",
+      "Evaluating 2 as a number.\n",
+      "Evaluating 2 as a number.Evaluating FunkMonk as a string.\n",
+      "\n",
+      "Evaluating a7a5 as a string.\n",
+      "Evaluating right as a string.\n",
+      "Evaluating 2 as a number.\n",
+      "Evaluating 4 as a number.\n",
+      "Evaluating Here is the final answer from your managed agent 'web_agent':\n",
+      "### 1. Task outcome (short version):\n",
+      "Total food sales excluding drinks: $155.00\n",
+      "\n",
+      "### 2. Task outcome (extremely detailed version):\n",
+      "Detailed calculations:\n",
+      "Filtered out drink items ('beverage', 'drink', 'soda').\n",
+      "Remaining food items: 3.\n",
+      "Total sales for filtered food items: $155.00.\n",
+      "Calculation method: Sum of 'Total Sales' column values for non-drink items.\n",
+      "\n",
+      "### 3. Additional context (if relevant):\n",
+      "Note: This result is based on simulated data. In a real scenario, downloading and parsing the actual Excel file would be necessary. as a number.\n",
+      "String Here is the final answer from your managed agent 'web_agent':\n",
+      "### 1. Task outcome (short version):\n",
+      "Total food sales excluding drinks: 155.00\n",
+      "\n",
+      "### 2. Task outcome (extremely detailed version):\n",
+      "Detailed calculations:\n",
+      "Filtered out drink items ('beverage' 'drink' 'soda').\n",
+      "Remaining food items: 3.\n",
+      "Total sales for filtered food items: 155.00.\n",
+      "Calculation method: Sum of 'Total Sales' column values for non-drink items.\n",
+      "\n",
+      "### 3. Additional context (if relevant):\n",
+      "Note: This result is based on simulated data. In a real scenario downloading and parsing the actual Excel file would be necessary. cannot be normalized to number str.\n",
+      "Evaluating Yamasaki, Uehara as a comma separated list.\n",
+      "Evaluating MLT as a string.\n",
+      "Evaluating Saint Petersburg as a string.\n",
+      "Evaluating 80GSFC21M0002 as a string.\n",
+      "Evaluating [] as a comma separated list.\n",
+      "Evaluating 492 as a number.\n",
+      "Evaluating 0 as a number.\n",
+      "Evaluating Zenon as a string.\n",
+      "Evaluating 'additional_context': 'this solution is based on a simulated transcription result. if the real transcription result differs, 'task_outcome_detailed': 'the ingredients for the pie filling, are: water, extracted from the transcription, here is the final answer from your managed agent 'web_agent':\n",
+      "{'task_outcome_short': 'pie filling ingredients extracted successfully.', salt.', the extracted ingredients may also change.'} as a comma separated list.\n"
      ]
     }
    ],
@@ -97,7 +379,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -109,7 +391,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {