| {"number": 0, "llm_response": "$22,500.00", "gold_answer": "$22,500.00", "prompt": "What is the total amount of the invoice?", "usage": {"input": 203, "metric": "tokens", "output": 11, "processing_time": 0.305250883102417, "total": 214}, "category": "core"} | |
| {"number": 1, "llm_response": "# 0001", "gold_answer": "#0001", "prompt": "What is the invoice number?", "usage": {"input": 200, "metric": "tokens", "output": 7, "processing_time": 0.19945549964904785, "total": 207}, "category": "core"} | |
| {"number": 2, "llm_response": "\u2022Front End Engineering Service - $5000.00;\r\n\u2022Back End Engineering Service - $7500.00;\r\n\u2022Quality Assurance Manager - $10,000.00;\r\n\u2022Total Amount - $22,500.00.", "gold_answer": "\u2022Front End Engineering Service;\n\u2022Back End Engineering Service;\n\u2022Quality Assurance Manager", "prompt": "What is a list of the items being purchased?", "usage": {"input": 203, "metric": "tokens", "output": 68, "processing_time": 1.7515947818756104, "total": 271}, "category": "core"} | |
| {"number": 3, "llm_response": "Bia Hermes", "gold_answer": "Bia Hermes", "prompt": "What is the name of the contact for question?", "usage": {"input": 203, "metric": "tokens", "output": 5, "processing_time": 0.15128159523010254, "total": 208}, "category": "core"} | |
| {"number": 4, "llm_response": "# 1000", "gold_answer": "#1000", "prompt": "What is the PO number?", "usage": {"input": 199, "metric": "tokens", "output": 7, "processing_time": 0.2009289264678955, "total": 206}, "category": "core"} | |
| {"number": 5, "llm_response": "Within 30 days of the invoice date.", "gold_answer": "within 30 days of 01/01/2022", "prompt": "When is payment due?", "usage": {"input": 198, "metric": "tokens", "output": 13, "processing_time": 0.35103321075439453, "total": 211}, "category": "core"} | |
| {"number": 6, "llm_response": "May 3, 2022", "gold_answer": "May 3, 2022", "prompt": "When is payment due?", "usage": {"input": 218, "metric": "tokens", "output": 10, "processing_time": 0.2775113582611084, "total": 228}, "category": "core"} | |
| {"number": 7, "llm_response": "$4,150.00", "gold_answer": "4150.00", "prompt": "What is the subtotal amount?", "usage": {"input": 220, "metric": "tokens", "output": 10, "processing_time": 0.27822065353393555, "total": 230}, "category": "core"} | |
| {"number": 8, "llm_response": "$4,565.00", "gold_answer": "4565.00", "prompt": "What is the total amount?", "usage": {"input": 219, "metric": "tokens", "output": 10, "processing_time": 0.2783362865447998, "total": 229}, "category": "core"} | |
| {"number": 9, "llm_response": "\u202230 boxes of printer paper;\r\n\u202210 printer cartridges;\r\n\u20225 laser printers.", "gold_answer": "\u202230 Boxes of Printer Paper;\n\u202210 Printer Cartridges;\n\u20225 Laser Printers.", "prompt": "What is a list of the purchased items?", "usage": {"input": 222, "metric": "tokens", "output": 25, "processing_time": 0.6605353355407715, "total": 247}, "category": "core"} | |
| {"number": 10, "llm_response": "234", "gold_answer": "234", "prompt": "What is the invoice number?", "usage": {"input": 220, "metric": "tokens", "output": 4, "processing_time": 0.12884831428527832, "total": 224}, "category": "core"} | |
| {"number": 11, "llm_response": "ABC Company and Delta Inc.", "gold_answer": "ABC Company and Delta Inc.", "prompt": "What is the name of the two companies?", "usage": {"input": 222, "metric": "tokens", "output": 8, "processing_time": 0.2289586067199707, "total": 230}, "category": "core"} | |
| {"number": 12, "llm_response": "30", "gold_answer": "30", "prompt": "How many boxes of paper are being purchased?", "usage": {"input": 222, "metric": "tokens", "output": 3, "processing_time": 0.10168671607971191, "total": 225}, "category": "core"} | |
| {"number": 13, "llm_response": "Due on receipt on May 3, 2022", "gold_answer": "Due on Receipt", "prompt": "What are the payment terms?", "usage": {"input": 219, "metric": "tokens", "output": 16, "processing_time": 0.4317505359649658, "total": 235}, "category": "core"} | |
| {"number": 14, "llm_response": "8675 River Run Road, Marlington, CT 09392", "gold_answer": "8675 River Run Road, Marlington, CT 09392", "prompt": "What is the address of ABC Company?", "usage": {"input": 221, "metric": "tokens", "output": 19, "processing_time": 0.5074400901794434, "total": 240}, "category": "core"} | |
| {"number": 15, "llm_response": "March 3, 2022", "gold_answer": "March 3, 2022", "prompt": "What is the date of the invoice?", "usage": {"input": 222, "metric": "tokens", "output": 11, "processing_time": 0.3041517734527588, "total": 233}, "category": "core"} | |
| {"number": 16, "llm_response": "January 20, 2021", "gold_answer": "January 20, 2021", "prompt": "When was Biden inaugurated?", "usage": {"input": 104, "metric": "tokens", "output": 12, "processing_time": 0.3082435131072998, "total": 116}, "category": "core"} | |
| {"number": 17, "llm_response": "Donald Trump", "gold_answer": "Donald Trump", "prompt": "Who was Biden's opponent in the 2020 presidential election?", "usage": {"input": 114, "metric": "tokens", "output": 3, "processing_time": 0.0824737548828125, "total": 117}, "category": "core"} | |
| {"number": 18, "llm_response": "Delaware", "gold_answer": "Delaware", "prompt": "What state is Biden from?", "usage": {"input": 104, "metric": "tokens", "output": 3, "processing_time": 0.08212065696716309, "total": 107}, "category": "core"} | |
| {"number": 19, "llm_response": "Barack Obama", "gold_answer": "Barack Obama", "prompt": "Who was President when Biden was Vice President?", "usage": {"input": 107, "metric": "tokens", "output": 4, "processing_time": 0.10780763626098633, "total": 111}, "category": "core"} | |
| {"number": 20, "llm_response": "1. Vivek Murthy;\r\n2. David A. Kessler;\r\n3. Marcella Nunez-Smith.", "gold_answer": "1. former Surgeon General Vivek Murthy;\n2. former FDA commissioner David A. Kessler;\n3. Yale University's Marcella Nunez-Smith.", "prompt": "What is a list of the three people that served as co-chairs of the COVID task force?", "usage": {"input": 362, "metric": "tokens", "output": 34, "processing_time": 0.9184415340423584, "total": 396}, "category": "core"} | |
| {"number": 21, "llm_response": "Jon Ossoff and Raphael Warnock", "gold_answer": "Jon Ossoff and Raphael Warnock", "prompt": "Who are the two people that won elections in Georgia?", "usage": {"input": 352, "metric": "tokens", "output": 13, "processing_time": 0.38178253173828125, "total": 365}, "category": "core"} | |
| {"number": 22, "llm_response": "Two-year term", "gold_answer": "Two-year term", "prompt": "What is the length of Raphael Warnock's term?", "usage": {"input": 356, "metric": "tokens", "output": 5, "processing_time": 0.17708158493041992, "total": 361}, "category": "core"} | |
| {"number": 23, "llm_response": "26,000", "gold_answer": "26,000", "prompt": "How many national guard members were deployed on January 6?", "usage": {"input": 353, "metric": "tokens", "output": 7, "processing_time": 0.2272651195526123, "total": 360}, "category": "core"} | |
| {"number": 24, "llm_response": "November 23, after Michigan certified its results.", "gold_answer": "November 23", "prompt": "When was the letter of ascertainment issued?", "usage": {"input": 351, "metric": "tokens", "output": 13, "processing_time": 0.37960267066955566, "total": 364}, "category": "core"} | |
| {"number": 25, "llm_response": "$52.9 billion", "gold_answer": "$52.9 billion", "prompt": "What was Microsoft's revenue in the 3rd quarter?", "usage": {"input": 201, "metric": "tokens", "output": 7, "processing_time": 0.20055770874023438, "total": 208}, "category": "core"} | |
| {"number": 26, "llm_response": "10%", "gold_answer": "Increased 10%, up 15% in constant currency", "prompt": "What was the improvement in operating income year-to-year?", "usage": {"input": 201, "metric": "tokens", "output": 4, "processing_time": 0.1243734359741211, "total": 205}, "category": "core"} | |
| {"number": 27, "llm_response": "March 31, 2023", "gold_answer": "March 31, 2023", "prompt": "When did the quarter end?", "usage": {"input": 194, "metric": "tokens", "output": 12, "processing_time": 0.3249344825744629, "total": 206}, "category": "core"} | |
| {"number": 28, "llm_response": "Focused execution by sales teams and partners in this dynamic environment.", "gold_answer": "Focused execution by our sales teams and partners", "prompt": "According to the CFO, what led to the increase in cloud revenue?", "usage": {"input": 171, "metric": "tokens", "output": 14, "processing_time": 0.36638760566711426, "total": 185}, "category": "core"} | |
| {"number": 29, "llm_response": "$28.5 billion, up 22% year-over-year.", "gold_answer": "$28.5 billion", "prompt": "What was the amount of Cloud revenue in the quarter?", "usage": {"input": 166, "metric": "tokens", "output": 19, "processing_time": 0.494248628616333, "total": 185}, "category": "core"} | |
| {"number": 30, "llm_response": "8%", "gold_answer": "10%", "prompt": "What was LinkedIn revenue growth in constant currency?", "usage": {"input": 448, "metric": "tokens", "output": 3, "processing_time": 0.13942933082580566, "total": 451}, "category": "core"} | |
| {"number": 31, "llm_response": "65.4 million", "gold_answer": "65.4 million", "prompt": "What is the number of Microsoft 365 consumer subscribers?", "usage": {"input": 453, "metric": "tokens", "output": 6, "processing_time": 0.21862244606018066, "total": 459}, "category": "core"} | |
| {"number": 32, "llm_response": "$22.1 billion", "gold_answer": "$22.1 billion", "prompt": "What was the amount of total revenue in Intelligent Cloud?", "usage": {"input": 451, "metric": "tokens", "output": 7, "processing_time": 0.2449817657470703, "total": 458}, "category": "core"} | |
| {"number": 33, "llm_response": "1956", "gold_answer": "1956", "prompt": "When was the field of AI founded?", "usage": {"input": 190, "metric": "tokens", "output": 5, "processing_time": 0.14237618446350098, "total": 195}, "category": "core"} | |
| {"number": 34, "llm_response": "The Church-Turing thesis suggests that a machine, by shuffling symbols as simple as \"0\" and \"1\", could simulate both mathematical deduction and formal reasoning.", "gold_answer": "A machine, by shuffling symbols as simple as '0' and '1' can simulate both mathematical deduction and formal reasoning.", "prompt": "What is the Church-Turing thesis?", "usage": {"input": 191, "metric": "tokens", "output": 38, "processing_time": 0.9781525135040283, "total": 229}, "category": "core"} | |
| {"number": 35, "llm_response": "1956", "gold_answer": "1943", "prompt": "When was the first paper on AI published?", "usage": {"input": 191, "metric": "tokens", "output": 5, "processing_time": 0.14288997650146484, "total": 196}, "category": "core"} | |
| {"number": 36, "llm_response": "1956", "gold_answer": "1956", "prompt": "When was the workshop at Dartmouth?", "usage": {"input": 191, "metric": "tokens", "output": 5, "processing_time": 0.1426386833190918, "total": 196}, "category": "core"} | |
| {"number": 37, "llm_response": "A period when obtaining funding for AI projects was difficult.", "gold_answer": "A period when obtaining funding for AI projects is difficult.", "prompt": "What is an AI winter?", "usage": {"input": 411, "metric": "tokens", "output": 12, "processing_time": 0.3686044216156006, "total": 423}, "category": "core"} | |
| {"number": 38, "llm_response": "Machines will be capable, within twenty years, of doing any work a man can do.", "gold_answer": "He predicted that machines would be capable of doing any work a man can do within twenty years.", "prompt": "What did Herbert Simon predict?", "usage": {"input": 411, "metric": "tokens", "output": 21, "processing_time": 0.6030068397521973, "total": 432}, "category": "core"} | |
| {"number": 39, "llm_response": "1987", "gold_answer": "1987.", "prompt": "When did the LISP machine market collapse?", "usage": {"input": 415, "metric": "tokens", "output": 5, "processing_time": 0.19104552268981934, "total": 420}, "category": "core"} | |
| {"number": 40, "llm_response": "TestCo Software, Inc.", "gold_answer": "TestCo Software, Inc.", "prompt": "What is the name of the Company?", "usage": {"input": 178, "metric": "tokens", "output": 7, "processing_time": 0.19146132469177246, "total": 185}, "category": "core"} | |
| {"number": 41, "llm_response": "Aphrodite Apollo (\"Executive\") and TestCo Software, Inc. (the \"Company\")", "gold_answer": "Aphrodite Apollo ('Executive') and TestCo Software, Inc. ('Company')", "prompt": "What are the names of the two parties?", "usage": {"input": 179, "metric": "tokens", "output": 24, "processing_time": 0.6227481365203857, "total": 203}, "category": "core"} | |
| {"number": 42, "llm_response": "April 16, 2012", "gold_answer": "April 16, 2012.", "prompt": "When will employment start?", "usage": {"input": 175, "metric": "tokens", "output": 12, "processing_time": 0.3175368309020996, "total": 187}, "category": "core"} | |
| {"number": 43, "llm_response": "Customer", "gold_answer": "Customer", "prompt": "Who owns the intellectual property?", "usage": {"input": 164, "metric": "tokens", "output": 2, "processing_time": 0.06323862075805664, "total": 166}, "category": "core"} | |
| {"number": 44, "llm_response": "30 days written notice", "gold_answer": "30 days", "prompt": "What is the notice period to terminate for convenience?", "usage": {"input": 151, "metric": "tokens", "output": 6, "processing_time": 0.1645066738128662, "total": 157}, "category": "core"} | |
| {"number": 45, "llm_response": "30 days written notice, if not cured within 15 days' notice of such breach.", "gold_answer": "If the breach is not cured within 15 days of notice of the breach.", "prompt": "When can termination after a material breach occur?", "usage": {"input": 151, "metric": "tokens", "output": 23, "processing_time": 0.5944380760192871, "total": 174}, "category": "core"} | |
| {"number": 46, "llm_response": "Yes, either party may terminate for convenience with 30 days written notice.", "gold_answer": "Yes, by either party with 30 days written notice.", "prompt": "Can the agreement be terminated?", "usage": {"input": 147, "metric": "tokens", "output": 18, "processing_time": 0.46987128257751465, "total": 165}, "category": "core"} | |
| {"number": 47, "llm_response": "Senior Vice President, Event Planning (\"SVP\") of the Employer's Workforce Optimization Division.", "gold_answer": "Senior Vice President, Event Planning ('SVP') of the Workforce Optimization Division.", "prompt": "What is the executive's role?", "usage": {"input": 380, "metric": "tokens", "output": 26, "processing_time": 0.7183167934417725, "total": 406}, "category": "core"} | |
| {"number": 48, "llm_response": "$200,000 annual rate", "gold_answer": "$200,000", "prompt": "What is the executive's base salary?", "usage": {"input": 190, "metric": "tokens", "output": 11, "processing_time": 0.29427456855773926, "total": 201}, "category": "core"} | |
| {"number": 49, "llm_response": "Section 2.2", "gold_answer": "Section 2.2", "prompt": "What section defines the base salary?", "usage": {"input": 188, "metric": "tokens", "output": 6, "processing_time": 0.16805577278137207, "total": 194}, "category": "core"} | |
| {"number": 50, "llm_response": "Annually", "gold_answer": "Annually", "prompt": "How often will the Board review the salary?", "usage": {"input": 190, "metric": "tokens", "output": 3, "processing_time": 0.09174799919128418, "total": 193}, "category": "core"} | |
| {"number": 51, "llm_response": "State of Massachusetts", "gold_answer": "State of Massachusetts, except for Section 18(a)", "prompt": "What is the governing law?", "usage": {"input": 320, "metric": "tokens", "output": 4, "processing_time": 0.14874696731567383, "total": 324}, "category": "core"} | |
| {"number": 52, "llm_response": "State of Delaware", "gold_answer": "State of Delaware", "prompt": "What law governs section 18(a)?", "usage": {"input": 325, "metric": "tokens", "output": 5, "processing_time": 0.17470622062683105, "total": 330}, "category": "core"} | |
| {"number": 53, "llm_response": "Section 19", "gold_answer": "Section 19", "prompt": "What section of the agreement defines the governing law?", "usage": {"input": 324, "metric": "tokens", "output": 5, "processing_time": 0.1734468936920166, "total": 329}, "category": "core"} | |
| {"number": 54, "llm_response": "195.12 points, or 0.76%", "gold_answer": "195.12 points", "prompt": "What was the gain on the DOW on Friday?", "usage": {"input": 528, "metric": "tokens", "output": 16, "processing_time": 0.4952065944671631, "total": 544}, "category": "core"} | |
| {"number": 55, "llm_response": "438,000", "gold_answer": "438,000 jobs", "prompt": "How many new jobs were added in August?", "usage": {"input": 526, "metric": "tokens", "output": 8, "processing_time": 0.2885923385620117, "total": 534}, "category": "core"} | |
| {"number": 56, "llm_response": "273,000", "gold_answer": "273,000 jobs", "prompt": "How many jobs were predicted by economists?", "usage": {"input": 526, "metric": "tokens", "output": 8, "processing_time": 0.28862786293029785, "total": 534}, "category": "core"} | |
| {"number": 57, "llm_response": "1.35%", "gold_answer": "Up 1.35%", "prompt": "What was the percentage in increase in the Nasdaq at closing?", "usage": {"input": 531, "metric": "tokens", "output": 6, "processing_time": 0.23895788192749023, "total": 537}, "category": "core"} | |
| {"number": 58, "llm_response": "Vibrant Industries Capital Advisors", "gold_answer": "Vibrant Industries Capital Advisors", "prompt": "Where does Margaret Jones work?", "usage": {"input": 523, "metric": "tokens", "output": 9, "processing_time": 0.315507173538208, "total": 532}, "category": "core"} | |
| {"number": 59, "llm_response": "4.58%", "gold_answer": "4.58%", "prompt": "What is the current rate on 10-year treasuries?", "usage": {"input": 532, "metric": "tokens", "output": 6, "processing_time": 0.23658227920532227, "total": 538}, "category": "core"} | |
| {"number": 60, "llm_response": "$350,000.", "gold_answer": "$350,000", "prompt": "What is the CEO's salary?", "usage": {"input": 49, "metric": "tokens", "output": 10, "processing_time": 0.257232666015625, "total": 59}, "category": "core"} | |
| {"number": 61, "llm_response": "$33.", "gold_answer": "$33", "prompt": "What is the stock price on Thursday?", "usage": {"input": 58, "metric": "tokens", "output": 5, "processing_time": 0.13238906860351562, "total": 63}, "category": "core"} | |
| {"number": 62, "llm_response": "61 years old.", "gold_answer": "61", "prompt": "What is Bob's age?", "usage": {"input": 42, "metric": "tokens", "output": 6, "processing_time": 0.15611839294433594, "total": 48}, "category": "core"} | |
| {"number": 63, "llm_response": "One Red River Road, Stamford, Connecticut 08234.", "gold_answer": "One Red River Road, Stamford, Connecticut 08234", "prompt": "What is the address of Red River Industries?", "usage": {"input": 128, "metric": "tokens", "output": 18, "processing_time": 0.46131014823913574, "total": 146}, "category": "core"} | |
| {"number": 64, "llm_response": "Simmons Corp.", "gold_answer": "Simmons Corp", "prompt": "At which company did Brian serve as head of Northeast Sales?", "usage": {"input": 75, "metric": "tokens", "output": 6, "processing_time": 0.15828442573547363, "total": 81}, "category": "core"} | |
| {"number": 65, "llm_response": "Vice President of Sales", "gold_answer": "Vice President of Sales", "prompt": "What is Mr. Williams role at Washtec Industries?", "usage": {"input": 75, "metric": "tokens", "output": 6, "processing_time": 0.15847539901733398, "total": 81}, "category": "core"} | |
| {"number": 66, "llm_response": "101% increase in revenue from the previous year.", "gold_answer": "Up 101%", "prompt": "What was the increase in revenue from the previous year?", "usage": {"input": 565, "metric": "tokens", "output": 13, "processing_time": 0.4236576557159424, "total": 578}, "category": "core"} | |
| {"number": 67, "llm_response": "$13.51 billion", "gold_answer": "$13.51 billion", "prompt": "What was the revenue in second quarter?", "usage": {"input": 562, "metric": "tokens", "output": 8, "processing_time": 0.2929270267486572, "total": 570}, "category": "core"} | |
| {"number": 68, "llm_response": "7.5 million shares", "gold_answer": "7.5 million shares", "prompt": "How many shares were repurchased in second quarter?", "usage": {"input": 565, "metric": "tokens", "output": 6, "processing_time": 0.24221396446228027, "total": 571}, "category": "core"} | |
| {"number": 69, "llm_response": "Jensen Huang", "gold_answer": "Jensen Huang", "prompt": "Who is the CEO of Nvidia?", "usage": {"input": 563, "metric": "tokens", "output": 5, "processing_time": 0.21592926979064941, "total": 568}, "category": "core"} | |
| {"number": 70, "llm_response": "141% increase.", "gold_answer": "Up 141% from Q1", "prompt": "What was the percentage increase in data center revenue compared to the first quarter?", "usage": {"input": 569, "metric": "tokens", "output": 7, "processing_time": 0.2685680389404297, "total": 576}, "category": "core"} | |
| {"number": 71, "llm_response": "$2.48", "gold_answer": "$2.48", "prompt": "What is GAAP earnings per share for the quarter?", "usage": {"input": 565, "metric": "tokens", "output": 6, "processing_time": 0.2422168254852295, "total": 571}, "category": "core"} | |
| {"number": 72, "llm_response": "September 28, 2023", "gold_answer": "September 28, 2023", "prompt": "When will the next dividend be paid?", "usage": {"input": 563, "metric": "tokens", "output": 12, "processing_time": 0.3999369144439697, "total": 575}, "category": "core"} | |
| {"number": 73, "llm_response": "148%", "gold_answer": "Up 148%", "prompt": "What is the percentage increase in non-GAAP earnings from the previous quarter?", "usage": {"input": 570, "metric": "tokens", "output": 5, "processing_time": 0.21615314483642578, "total": 575}, "category": "core"} | |
| {"number": 74, "llm_response": "Yes, 71.5% > 70%.", "gold_answer": "Yes, between 71.5% and 72.%", "prompt": "Is the expected gross margin greater than 70%", "usage": {"input": 325, "metric": "tokens", "output": 14, "processing_time": 0.40485382080078125, "total": 339}, "category": "core"} | |
| {"number": 75, "llm_response": "$16.00 billion, plus or minus 2%.", "gold_answer": "$16.00 billion, plus or minus 2%.", "prompt": "What is the expected revenue for third quarter?", "usage": {"input": 323, "metric": "tokens", "output": 16, "processing_time": 0.4576704502105713, "total": 339}, "category": "core"} | |
| {"number": 76, "llm_response": "The NVIDIA GH200 Grace Hopper Superchip for complex AI and HPC workloads.", "gold_answer": "NVIDIA GH200 Grace Hopper Superchip", "prompt": "Which new product is shipping in the quarter?", "usage": {"input": 323, "metric": "tokens", "output": 24, "processing_time": 0.6656534671783447, "total": 347}, "category": "core"} | |
| {"number": 77, "llm_response": "14.5% plus or minus 1%", "gold_answer": "14.5%, plus or minor 1%", "prompt": "What is the forecast tax rate?", "usage": {"input": 321, "metric": "tokens", "output": 13, "processing_time": 0.37952518463134766, "total": 334}, "category": "core"} | |
| {"number": 78, "llm_response": "$2.00 billion", "gold_answer": "$2.00 billion", "prompt": "What is the amount of expected non-GAAP operating expense?", "usage": {"input": 327, "metric": "tokens", "output": 7, "processing_time": 0.22735977172851562, "total": 334}, "category": "core"} | |
| {"number": 79, "llm_response": "$2.95 billion", "gold_answer": "$2.95 billion", "prompt": "What is the amount of expected GAAP operating expense?", "usage": {"input": 325, "metric": "tokens", "output": 7, "processing_time": 0.23014497756958008, "total": 332}, "category": "core"} | |
| {"number": 80, "llm_response": "$2.49 billion, up 11% from the previous quarter and up 22% from the prior year.", "gold_answer": "$2.49 billion", "prompt": "What was gaming revenue in the quarter?", "usage": {"input": 406, "metric": "tokens", "output": 28, "processing_time": 0.7810342311859131, "total": 434}, "category": "core"} | |
| {"number": 81, "llm_response": "$379 million, up 28% from the previous quarter, and down 24% from a year ago.", "gold_answer": "$379 million", "prompt": "What was professional visualization revenue in the quarter?", "usage": {"input": 408, "metric": "tokens", "output": 28, "processing_time": 0.7880549430847168, "total": 436}, "category": "core"} | |
| {"number": 82, "llm_response": "28% increase from the previous quarter.", "gold_answer": "Up 28%", "prompt": "What was the increase in professional visualization revenue quarter-to-quarter?", "usage": {"input": 414, "metric": "tokens", "output": 10, "processing_time": 0.31851625442504883, "total": 424}, "category": "core"} | |
| {"number": 83, "llm_response": "Pixar, Adobe, Apple and Autodesk.", "gold_answer": "Pixar, Adobe, Apple, Autodesk and NVIDIA", "prompt": "Which companies are participating in the Alliance for OpenUSD?", "usage": {"input": 410, "metric": "tokens", "output": 14, "processing_time": 0.42180943489074707, "total": 424}, "category": "core"} | |
| {"number": 84, "llm_response": "35", "gold_answer": "35", "prompt": "How many DLSS games were added?", "usage": {"input": 406, "metric": "tokens", "output": 3, "processing_time": 0.13766694068908691, "total": 409}, "category": "core"} | |
| {"number": 85, "llm_response": "$299", "gold_answer": "$299", "prompt": "What is the lowest price for GeForce RTX 4060 family?", "usage": {"input": 415, "metric": "tokens", "output": 5, "processing_time": 0.1909623146057129, "total": 420}, "category": "core"} | |
| {"number": 86, "llm_response": "MediaTek", "gold_answer": "MediaTek", "prompt": "Who is NVIDIA's partner for the driver assistance system?", "usage": {"input": 127, "metric": "tokens", "output": 4, "processing_time": 0.10899734497070312, "total": 131}, "category": "core"} | |
| {"number": 87, "llm_response": "Refinitiv", "gold_answer": "Refinitiv", "prompt": "What is the former name of LSEG?", "usage": {"input": 472, "metric": "tokens", "output": 4, "processing_time": 0.16846275329589844, "total": 476}, "category": "core"} | |
| {"number": 88, "llm_response": "Jerome Powell", "gold_answer": "Jerome Powell", "prompt": "Who is the chairman of the Federal Reserve?", "usage": {"input": 471, "metric": "tokens", "output": 6, "processing_time": 0.22310113906860352, "total": 477}, "category": "core"} | |
| {"number": 89, "llm_response": "Japan", "gold_answer": "Japan", "prompt": "Which country had a trade surplus in September?", "usage": {"input": 170, "metric": "tokens", "output": 3, "processing_time": 0.09005618095397949, "total": 173}, "category": "core"} | |
| {"number": 90, "llm_response": "16.3% year-on-year decline, compared to the same period last year.", "gold_answer": "Down 16.3%", "prompt": "What was the decline in imports in September in Japan?", "usage": {"input": 171, "metric": "tokens", "output": 21, "processing_time": 0.5461428165435791, "total": 192}, "category": "core"} | |
| {"number": 91, "llm_response": "$416.6 million", "gold_answer": "62.4 billion yen ($416.6 million)", "prompt": "What was the trade surplus?", "usage": {"input": 167, "metric": "tokens", "output": 8, "processing_time": 0.21625518798828125, "total": 175}, "category": "core"} | |
| {"number": 92, "llm_response": "$416.6 million", "gold_answer": "Deficit of 42.5 billion yen.", "prompt": "What did economists expect for the trade surplus amount?", "usage": {"input": 172, "metric": "tokens", "output": 8, "processing_time": 0.21763205528259277, "total": 180}, "category": "core"} | |
| {"number": 93, "llm_response": "4.3% year on year", "gold_answer": "Up 4.3%", "prompt": "What was the increase in exports?", "usage": {"input": 167, "metric": "tokens", "output": 8, "processing_time": 0.22057485580444336, "total": 175}, "category": "core"} | |
| {"number": 94, "llm_response": "S&P 500's bear market bottom of 3,577.", "gold_answer": "3,577", "prompt": "What was the S&P trading level one year ago?", "usage": {"input": 748, "metric": "tokens", "output": 21, "processing_time": 0.6594524383544922, "total": 769}, "category": "core"} | |
| {"number": 95, "llm_response": "3.2%", "gold_answer": "3.2%", "prompt": "What was the cost of living adjustment for 2024?", "usage": {"input": 751, "metric": "tokens", "output": 5, "processing_time": 0.24117422103881836, "total": 756}, "category": "core"} | |
| {"number": 96, "llm_response": "Buy", "gold_answer": "Buy", "prompt": "What is Bank of America's rating on Target?", "usage": {"input": 747, "metric": "tokens", "output": 3, "processing_time": 0.18806076049804688, "total": 750}, "category": "core"} | |
| {"number": 97, "llm_response": "Buy rating.", "gold_answer": "Buy", "prompt": "What is Bank of Americas' rating on Snowflake?", "usage": {"input": 749, "metric": "tokens", "output": 5, "processing_time": 0.24349713325500488, "total": 754}, "category": "core"} | |
| {"number": 98, "llm_response": "$16 per share, down from $17.", "gold_answer": "$16", "prompt": "What is Barclay's new price target on UTZ?", "usage": {"input": 751, "metric": "tokens", "output": 13, "processing_time": 0.45447397232055664, "total": 764}, "category": "core"} | |
| {"number": 99, "llm_response": "133 million euros, down 69% year-on-year.", "gold_answer": "133 million euros.", "prompt": "What was third quarter profit?", "usage": {"input": 138, "metric": "tokens", "output": 19, "processing_time": 0.4968373775482178, "total": 157}, "category": "core"} | |
| {"number": 100, "llm_response": "Not Found.", "gold_answer": "Not Found", "prompt": "What is the purchase order number?", "usage": {"input": 220, "metric": "tokens", "output": 4, "processing_time": 0.1279616355895996, "total": 224}, "category": "not_found_classification"} | |
| {"number": 101, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "Who is the EVP of Human Resources?", "usage": {"input": 164, "metric": "tokens", "output": 4, "processing_time": 0.11468315124511719, "total": 168}, "category": "not_found_classification"} | |
| {"number": 102, "llm_response": "Not Found.", "gold_answer": "Not Found", "prompt": "What is the annual bonus amount?", "usage": {"input": 188, "metric": "tokens", "output": 4, "processing_time": 0.11833715438842773, "total": 192}, "category": "not_found_classification"} | |
| {"number": 103, "llm_response": "Not Found.", "gold_answer": "Not Found", "prompt": "What is the address of Simmons Technologies?", "usage": {"input": 127, "metric": "tokens", "output": 4, "processing_time": 0.10862851142883301, "total": 131}, "category": "not_found_classification"} | |
| {"number": 104, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "What was the amount of free cash flow in the quarter?", "usage": {"input": 566, "metric": "tokens", "output": 4, "processing_time": 0.1891465187072754, "total": 570}, "category": "not_found_classification"} | |
| {"number": 105, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "What is the expected revenue for fourth quarter?", "usage": {"input": 323, "metric": "tokens", "output": 4, "processing_time": 0.15070319175720215, "total": 327}, "category": "not_found_classification"} | |
| {"number": 106, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "When was the A100 chip released?", "usage": {"input": 324, "metric": "tokens", "output": 4, "processing_time": 0.15001320838928223, "total": 328}, "category": "not_found_classification"} | |
| {"number": 107, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "What was gaming revenue for the quarter?", "usage": {"input": 121, "metric": "tokens", "output": 4, "processing_time": 0.10767674446105957, "total": 125}, "category": "not_found_classification"} | |
| {"number": 108, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "What was third quarter revenue?", "usage": {"input": 119, "metric": "tokens", "output": 4, "processing_time": 0.1079871654510498, "total": 123}, "category": "not_found_classification"} | |
| {"number": 109, "llm_response": "Not Found.", "gold_answer": "Not Found", "prompt": "What is gaming revenue for the full year FY24?", "usage": {"input": 411, "metric": "tokens", "output": 4, "processing_time": 0.16482186317443848, "total": 415}, "category": "not_found_classification"} | |
| {"number": 110, "llm_response": "Elon Musk", "gold_answer": "Not Found.", "prompt": "Who is the CEO of Netflix?", "usage": {"input": 469, "metric": "tokens", "output": 5, "processing_time": 0.19659137725830078, "total": 474}, "category": "not_found_classification"} | |
| {"number": 111, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "What was China's trade balance in September?", "usage": {"input": 170, "metric": "tokens", "output": 4, "processing_time": 0.11531352996826172, "total": 174}, "category": "not_found_classification"} | |
| {"number": 112, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "What was Japan's trade balance in March?", "usage": {"input": 170, "metric": "tokens", "output": 4, "processing_time": 0.1148674488067627, "total": 174}, "category": "not_found_classification"} | |
| {"number": 113, "llm_response": "Not Found.", "gold_answer": "Not Found", "prompt": "What was the cost of living adjustment for 2023?", "usage": {"input": 751, "metric": "tokens", "output": 4, "processing_time": 0.21527647972106934, "total": 755}, "category": "not_found_classification"} | |
| {"number": 114, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "When is TJX reporting earnings?", "usage": {"input": 744, "metric": "tokens", "output": 4, "processing_time": 0.21236896514892578, "total": 748}, "category": "not_found_classification"} | |
| {"number": 115, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "How many were laptops were purchased?", "usage": {"input": 201, "metric": "tokens", "output": 4, "processing_time": 0.12601470947265625, "total": 205}, "category": "not_found_classification"} | |
| {"number": 116, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "Who is the secretary of agriculture?", "usage": {"input": 105, "metric": "tokens", "output": 4, "processing_time": 0.10798311233520508, "total": 109}, "category": "not_found_classification"} | |
| {"number": 117, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "What is the executive's salary?", "usage": {"input": 380, "metric": "tokens", "output": 4, "processing_time": 0.15429472923278809, "total": 384}, "category": "not_found_classification"} | |
| {"number": 118, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "What is the 4th quarter revenue of Broadcom?", "usage": {"input": 566, "metric": "tokens", "output": 4, "processing_time": 0.190871000289917, "total": 570}, "category": "not_found_classification"} | |
| {"number": 119, "llm_response": "Not Found.", "gold_answer": "Not Found.", "prompt": "How old is Maribeth?", "usage": {"input": 43, "metric": "tokens", "output": 4, "processing_time": 0.11010956764221191, "total": 47}, "category": "not_found_classification"} | |
| {"number": 120, "llm_response": "Yes.", "gold_answer": "Yes, by more than 22%.", "prompt": "Has the S&P increased over the last year?", "usage": {"input": 747, "metric": "tokens", "output": 3, "processing_time": 0.19014811515808105, "total": 750}, "category": "boolean_mc"} | |
| {"number": 121, "llm_response": "No.", "gold_answer": "No, Chipotle is increasing prices.", "prompt": "Is Chipotle lowering prices?", "usage": {"input": 745, "metric": "tokens", "output": 3, "processing_time": 0.189955472946167, "total": 748}, "category": "boolean_mc"} | |
| {"number": 122, "llm_response": "No, Barclay's lowered its price target on KHC to $38.", "gold_answer": "No, it is lowering the target from $38 to $36.", "prompt": "Is Barclay's raising its price target on KHC?", "usage": {"input": 750, "metric": "tokens", "output": 20, "processing_time": 0.634990930557251, "total": 770}, "category": "boolean_mc"} | |
| {"number": 123, "llm_response": "No, 4.98 billion euros < 5 billion euros.", "gold_answer": "No.", "prompt": "Were third quarter sales more than 5 billion euros?", "usage": {"input": 145, "metric": "tokens", "output": 18, "processing_time": 0.4678785800933838, "total": 163}, "category": "boolean_mc"} | |
| {"number": 124, "llm_response": "Yes.", "gold_answer": "Yes.", "prompt": "Did the trade surplus beat expectations?", "usage": {"input": 168, "metric": "tokens", "output": 3, "processing_time": 0.0896613597869873, "total": 171}, "category": "boolean_mc"} | |
| {"number": 125, "llm_response": "No, Dow Jones futures declined 73 points, or 0.2%.", "gold_answer": "No, down 73 points.", "prompt": "Did Dow Jones futures increase?", "usage": {"input": 469, "metric": "tokens", "output": 19, "processing_time": 0.5578157901763916, "total": 488}, "category": "boolean_mc"} | |
| {"number": 126, "llm_response": "No, the yield on the 10-year is 4.9%.", "gold_answer": "Yes, it topped 4.9%.", "prompt": "Is the yield on the 10-year above 4.8%?", "usage": {"input": 479, "metric": "tokens", "output": 18, "processing_time": 0.5327181816101074, "total": 497}, "category": "boolean_mc"} | |
| {"number": 127, "llm_response": "No, the Nasdaq Composite was the underperformer, off 1.6%.", "gold_answer": "No, it was down 1.6%.", "prompt": "Did the Nasdaq Composite increase?", "usage": {"input": 471, "metric": "tokens", "output": 21, "processing_time": 0.6085350513458252, "total": 492}, "category": "boolean_mc"} | |
| {"number": 128, "llm_response": "No, revenue was down 15%.", "gold_answer": "No, it was down 15% from the previous quarter.", "prompt": "Did automotive revenue increase in the current quarter compared to last quarter?", "usage": {"input": 127, "metric": "tokens", "output": 10, "processing_time": 0.26063990592956543, "total": 137}, "category": "boolean_mc"} | |
| {"number": 129, "llm_response": "Yes, up 15%.", "gold_answer": "Yes, up 15%.", "prompt": "Did automotive revenue increase year-over-year?", "usage": {"input": 124, "metric": "tokens", "output": 8, "processing_time": 0.20961689949035645, "total": 132}, "category": "boolean_mc"} | |
| {"number": 130, "llm_response": "No, $253 million < $250 million.", "gold_answer": "Yes", "prompt": "Was automotive revenue more than $250 million?", "usage": {"input": 125, "metric": "tokens", "output": 15, "processing_time": 0.38651561737060547, "total": 140}, "category": "boolean_mc"} | |
| {"number": 131, "llm_response": "No.", "gold_answer": "No, it was up 11% quarter-to-quarter and 22% year-to-year.", "prompt": "Did gaming revenue decrease?", "usage": {"input": 403, "metric": "tokens", "output": 3, "processing_time": 0.13942241668701172, "total": 406}, "category": "boolean_mc"} | |
| {"number": 132, "llm_response": "Yes, 101% > 50%.", "gold_answer": "Yes, it increased 101% year-to-year.", "prompt": "Did revenue increase more than 50% year-to-year?", "usage": {"input": 569, "metric": "tokens", "output": 13, "processing_time": 0.430039644241333, "total": 582}, "category": "boolean_mc"} | |
| {"number": 133, "llm_response": "Yes, $3.28 billion > $3.00 billion.", "gold_answer": "Yes, $3.38 billion was returned", "prompt": "Was more than $3 billion returned to shareholders in the quarter?", "usage": {"input": 568, "metric": "tokens", "output": 17, "processing_time": 0.5278074741363525, "total": 585}, "category": "boolean_mc"} | |
| {"number": 134, "llm_response": "Not Found.", "gold_answer": "No, it decreased by 28%", "prompt": "Did Windows OEM revenue increase?", "usage": {"input": 321, "metric": "tokens", "output": 4, "processing_time": 0.15083742141723633, "total": 325}, "category": "boolean_mc"} | |
| {"number": 135, "llm_response": "No.", "gold_answer": "No, net sales declined 20% year-on-year, and the company has described the market environment as 'challenging.'", "prompt": "Are Nokia's products performing well in the market?", "usage": {"input": 145, "metric": "tokens", "output": 3, "processing_time": 0.08874106407165527, "total": 148}, "category": "boolean_mc"} | |
| {"number": 136, "llm_response": "Yes.", "gold_answer": "Yes, net sales declined 20% year-on-year.", "prompt": "Is Nokia having trouble with sales?", "usage": {"input": 141, "metric": "tokens", "output": 3, "processing_time": 0.09184384346008301, "total": 144}, "category": "boolean_mc"} | |
| {"number": 137, "llm_response": "No.", "gold_answer": "No, exports to Asia decreased for the last nine months.", "prompt": "Are Japan's exports to Asia increasing?", "usage": {"input": 169, "metric": "tokens", "output": 3, "processing_time": 0.09014630317687988, "total": 172}, "category": "boolean_mc"} | |
| {"number": 138, "llm_response": "Yes, exports were up 4.3% year on year.", "gold_answer": "Yes, by 4.3% year to year.", "prompt": "Are Japan's total exports increasing?", "usage": {"input": 168, "metric": "tokens", "output": 15, "processing_time": 0.3937687873840332, "total": 183}, "category": "boolean_mc"} | |
| {"number": 139, "llm_response": "No.", "gold_answer": "No, the base salary is $200,000.", "prompt": "Is the executive's base salary $250,000?", "usage": {"input": 197, "metric": "tokens", "output": 3, "processing_time": 0.1010127067565918, "total": 200}, "category": "boolean_mc"} | |
| {"number": 140, "llm_response": "$253 million + $100 million = $353 million.", "gold_answer": "$353 million", "prompt": "If automotive revenue increases by $100 million in third quarter, what will be the amount?", "usage": {"input": 134, "metric": "tokens", "output": 19, "processing_time": 0.4968092441558838, "total": 153}, "category": "math_basic"} | |
| {"number": 141, "llm_response": "$14 + $2 = $16.", "gold_answer": "$16", "prompt": "If the stock increases by $2, what is the new price?", "usage": {"input": 33, "metric": "tokens", "output": 12, "processing_time": 0.30364370346069336, "total": 45}, "category": "math_basic"} | |
| {"number": 142, "llm_response": "$12 - $3 = $9. The new price is $9.", "gold_answer": "$9", "prompt": "If the stock decreases by $3, what is the new price?", "usage": {"input": 41, "metric": "tokens", "output": 19, "processing_time": 0.4807167053222656, "total": 60}, "category": "math_basic"} | |
| {"number": 143, "llm_response": "January 15, 2019.", "gold_answer": "January 15, 2019", "prompt": "What is the date when the loan will expire?", "usage": {"input": 46, "metric": "tokens", "output": 13, "processing_time": 0.3326447010040283, "total": 59}, "category": "math_basic"} | |
| {"number": 144, "llm_response": "37 + 10 = 47. Bryan is now 47 years old.", "gold_answer": "47 years old.", "prompt": "If the year is now 2022, how old is Bryan now?", "usage": {"input": 40, "metric": "tokens", "output": 22, "processing_time": 0.5623486042022705, "total": 62}, "category": "math_basic"} | |
| {"number": 145, "llm_response": "$425,000 + $25,000 = $450,000.", "gold_answer": "$450,000", "prompt": "What is the amount of the executive's new salary?", "usage": {"input": 53, "metric": "tokens", "output": 27, "processing_time": 0.6856160163879395, "total": 80}, "category": "math_basic"} | |
| {"number": 146, "llm_response": "January 15, 3 days after the invoice was issued on January 12.", "gold_answer": "January 15", "prompt": "When is the last day to pay the invoice?", "usage": {"input": 45, "metric": "tokens", "output": 22, "processing_time": 0.5589051246643066, "total": 67}, "category": "math_basic"} | |
| {"number": 147, "llm_response": "1,000 X 2 = 2,000.", "gold_answer": "2,000", "prompt": "How many shares traded on Wednesday?", "usage": {"input": 51, "metric": "tokens", "output": 17, "processing_time": 0.4359462261199951, "total": 68}, "category": "math_basic"} | |
| {"number": 148, "llm_response": "$1200 X 50% = $600. The new rental price will be $1200 + $600 = $1800.", "gold_answer": "$1800", "prompt": "If the rent increased by 50%, what is the new rental price?", "usage": {"input": 44, "metric": "tokens", "output": 41, "processing_time": 1.0360450744628906, "total": 85}, "category": "math_basic"} | |
| {"number": 149, "llm_response": "$200,000 X 50% = $100,000.", "gold_answer": "$100,000", "prompt": "What is the amount of the target bonus?", "usage": {"input": 54, "metric": "tokens", "output": 24, "processing_time": 0.6074721813201904, "total": 78}, "category": "math_basic"} | |
| {"number": 150, "llm_response": "1. Liam - 41 years old;\r\n2. Myra - 23 years old;\r\n3. Ashish - 27 years old;\r\n4. Francois - 29 years old.", "gold_answer": "1. Liam - 41;\n2. Francois - 29;\n3. Ashish - 27;\n4. Myra - 23.", "prompt": "What is a list of the people ranked in order of age, from oldest to youngest?", "usage": {"input": 64, "metric": "tokens", "output": 52, "processing_time": 1.3145341873168945, "total": 116}, "category": "math_basic"} | |
| {"number": 151, "llm_response": "John is 47 years old, and Elizabeth is 35 years old, so John is older.", "gold_answer": "John", "prompt": "Who is older - Elizabeth or John?", "usage": {"input": 34, "metric": "tokens", "output": 23, "processing_time": 0.5810141563415527, "total": 57}, "category": "math_basic"} | |
| {"number": 152, "llm_response": "Chocolate - $3.25.", "gold_answer": "Chocolate - $3.25", "prompt": "Which item has the lowest price?", "usage": {"input": 56, "metric": "tokens", "output": 10, "processing_time": 0.25815415382385254, "total": 66}, "category": "math_basic"} | |
| {"number": 153, "llm_response": "Coffee - $5.50.", "gold_answer": "Cheese - $6.25", "prompt": "Which item has the highest price?", "usage": {"input": 56, "metric": "tokens", "output": 11, "processing_time": 0.2836596965789795, "total": 67}, "category": "math_basic"} | |
| {"number": 154, "llm_response": "No, chocolate is $3.25 and coffee is $5.50.", "gold_answer": "No, chocolate has a lower price than coffee.", "prompt": "Does chocolate have a higher price than coffee?", "usage": {"input": 54, "metric": "tokens", "output": 19, "processing_time": 0.48618507385253906, "total": 73}, "category": "math_basic"} | |
| {"number": 155, "llm_response": "Nasdaq 100 futures.", "gold_answer": "Nasdaq 100 futures, up 63 points.", "prompt": "Which stock index increased by the most points yesterday?", "usage": {"input": 105, "metric": "tokens", "output": 12, "processing_time": 0.3100717067718506, "total": 117}, "category": "math_basic"} | |
| {"number": 156, "llm_response": "Nasdaq 100 futures, up 0.5%.", "gold_answer": "S&P 500 futures futures, up 1.2%.", "prompt": "Which stock index increased by the highest percentage yesterday?", "usage": {"input": 105, "metric": "tokens", "output": 18, "processing_time": 0.4607834815979004, "total": 123}, "category": "math_basic"} | |
| {"number": 157, "llm_response": "Red Dog LLC - $61.", "gold_answer": "River Industries - $53.", "prompt": "Which stock has the highest price?", "usage": {"input": 94, "metric": "tokens", "output": 9, "processing_time": 0.23327231407165527, "total": 103}, "category": "math_basic"} | |
| {"number": 158, "llm_response": "$19 + $2 = $21.", "gold_answer": "$21", "prompt": "If Smithson stock increases by $2, what is the new price?", "usage": {"input": 89, "metric": "tokens", "output": 12, "processing_time": 0.30855560302734375, "total": 101}, "category": "math_basic"} | |
| {"number": 159, "llm_response": "PO 1993 - $87,000.", "gold_answer": "PO 1993", "prompt": "Which PO number has the smallest value?", "usage": {"input": 127, "metric": "tokens", "output": 16, "processing_time": 0.4140019416809082, "total": 143}, "category": "math_basic"} | |
| {"number": 160, "llm_response": "(C) $17.5 billion.", "gold_answer": "(D) $28.5 billion", "prompt": "What was the amount of Cloud revenue in the quarter - (A) $52.9 billion, (B) $18.3 billion, (C) $17.5 billion, or (D) $28.5 billion?", "usage": {"input": 709, "metric": "tokens", "output": 11, "processing_time": 0.39849042892456055, "total": 720}, "category": "complex_qa"} | |
| {"number": 161, "llm_response": "(B) TestCo Software.", "gold_answer": "(B) TestCo Software.", "prompt": "What is the name of the company - (A) Microsoft, (B) TestCo Software, (C) Aphrodite Apollo, or (D) Executive Employment Agreement?", "usage": {"input": 210, "metric": "tokens", "output": 8, "processing_time": 0.22780966758728027, "total": 218}, "category": "complex_qa"} | |
| {"number": 162, "llm_response": "(D) $10,000.", "gold_answer": "(D) $10,000.", "prompt": "What is the price for the services of the Quality Assurance Manager - (A) $1,000, (B) $7500, (C) $22,500, or (D) $10,000?", "usage": {"input": 249, "metric": "tokens", "output": 12, "processing_time": 0.3366739749908447, "total": 261}, "category": "complex_qa"} | |
| {"number": 163, "llm_response": "(C) 14%.", "gold_answer": "(C) 14%.", "prompt": "What was the percentage increase in diluted earnings per share in constant currency - (A) 10%, (B) 7%, (C) 14%, or (D) 15%?", "usage": {"input": 233, "metric": "tokens", "output": 8, "processing_time": 0.23274707794189453, "total": 241}, "category": "complex_qa"} | |
| {"number": 164, "llm_response": "(B) 69%.", "gold_answer": "(B) 69%.", "prompt": "What is the rate of decline in profit - (A) 20%, (B) 69%, (C) 68%, (D) 32%, or (E) 133 million?", "usage": {"input": 179, "metric": "tokens", "output": 8, "processing_time": 0.21804046630859375, "total": 187}, "category": "complex_qa"} | |
| {"number": 165, "llm_response": "\u20222019 - $24,578 million;\r\n\u20222020 - $31,536 million;\r\n\u20222021 - $53,823 million;\r\n\u20222022 - $81,462 million.", "gold_answer": "\u20222019 - 24,578;\n\u20222020 - 31,536;\n\u20222021 - 53,823;\n\u20222022 - 81,462.", "prompt": "What is a list of net sales in 2019, 2020, 2021, and 2022?", "usage": {"input": 888, "metric": "tokens", "output": 64, "processing_time": 1.8479325771331787, "total": 952}, "category": "complex_qa"} | |
| {"number": 166, "llm_response": "$297", "gold_answer": "297.", "prompt": "What is non-operating income interest in 2022?", "usage": {"input": 870, "metric": "tokens", "output": 5, "processing_time": 0.2673029899597168, "total": 875}, "category": "complex_qa"} | |
| {"number": 167, "llm_response": "$6,496", "gold_answer": "6,496.", "prompt": "What is Operating Income in 2021?", "usage": {"input": 868, "metric": "tokens", "output": 7, "processing_time": 0.3192770481109619, "total": 875}, "category": "complex_qa"} | |
| {"number": 168, "llm_response": "$24,578", "gold_answer": "18,402.", "prompt": "What is the cost of goods sold (excluding depreciation) in 2019?", "usage": {"input": 878, "metric": "tokens", "output": 8, "processing_time": 0.3466513156890869, "total": 886}, "category": "complex_qa"} | |
| {"number": 169, "llm_response": "$13,719", "gold_answer": "13,719", "prompt": "What is the amount of pretax income in 2022?", "usage": {"input": 870, "metric": "tokens", "output": 8, "processing_time": 0.3468756675720215, "total": 878}, "category": "complex_qa"} | |
| {"number": 170, "llm_response": "\u2022Herbert Simon\r\n\u2022Marvin Minsky\r\n\u2022Papert\r\n\u2022Sir James Lighthill\r\n\u2022Japan's Fifth Generation Computer Project\r\n\u2022U.S. and British governments\r\n\u2022U.S. Congress", "gold_answer": "\u2022Herbert Simon;\n\u2022Marvin Minsky;\n\u2022Sir James Lighthill;\n\u2022Papert.", "prompt": "What is a list of all of the people mentioned?", "usage": {"input": 416, "metric": "tokens", "output": 55, "processing_time": 1.4838297367095947, "total": 471}, "category": "complex_qa"} | |
| {"number": 171, "llm_response": "\u2022ServiceNow and Accenture,\r\n\u2022VMware,\r\n\u2022Snowflake,\r\n\u2022WPP,\r\n\u2022Softbank, and\r\n\u2022Hugging Face.", "gold_answer": "\u2022Amazon Web Services;\n\u2022Microsoft Azure;\u2022\nRegional Cloud Service Providers;\n\u2022ServiceNow;\n\u2022Accenture;\n\u2022VMWare;\n\u2022Snowflake;\n\u2022WPP;\n\u2022Softbank;\n\u2022Hugging Face.", "prompt": "What is a list of Nvidia's partners mentioned in the text?", "usage": {"input": 613, "metric": "tokens", "output": 40, "processing_time": 1.1449174880981445, "total": 653}, "category": "complex_qa"} | |
| {"number": 172, "llm_response": "\u2022Tesla,\r\n\u2022Netflix,\r\n\u2022Fifth Third,\r\n\u2022KeyCorp,\r\n\u2022Union Pacific,\r\n\u2022CSX, and\r\n\u2022American Airlines.", "gold_answer": "\u2022Tesla;\n;\u2022Netflix;\n\u2022KeyCorp;\n\u2022Fifth Third;\n\u2022Truist Financial;\n\u2022American Airlines;\n\u2022Union Pacific;\n\u2022CSX.", "prompt": "What is a list of the companies that are mentioned in the text?", "usage": {"input": 476, "metric": "tokens", "output": 41, "processing_time": 1.1308844089508057, "total": 517}, "category": "complex_qa"} | |
| {"number": 173, "llm_response": "U.S., Britain, Japan.", "gold_answer": "1. U.S;\n2. UK;\n3. Japan.", "prompt": "Which three governments are mentioned in the text?", "usage": {"input": 414, "metric": "tokens", "output": 9, "processing_time": 0.2971820831298828, "total": 423}, "category": "complex_qa"} | |
| {"number": 174, "llm_response": "1. The softer wage number in the jobs report may have made investors rethink their earlier bearish stance;\r\n2. The pullback in yields from the day's highs;\r\n3. The market had gotten extremely oversold, with the S&P 500 down more than 9% from its high earlier this year.", "gold_answer": "Traders were unclear of the reason, but some noted it could be the softer wage numberin the jobs report.", "prompt": "Why did the intraday reversal occur?", "usage": {"input": 527, "metric": "tokens", "output": 76, "processing_time": 2.066423177719116, "total": 603}, "category": "complex_qa"} | |
| {"number": 175, "llm_response": "Nokia said that it will reduce its cost base and increase operation efficiency to \"address the challenging market environment.\"", "gold_answer": "Cost cutting plan", "prompt": "Why is Nokia cutting jobs?", "usage": {"input": 140, "metric": "tokens", "output": 24, "processing_time": 0.6234850883483887, "total": 164}, "category": "complex_qa"} | |
| {"number": 176, "llm_response": "Tesla stock declined 4% after the company missed expectations on earnings and revenue in the third quarter. CEO Elon Musk also warned that the Cybertruck will not produce much positive cash flow more than a year after production starts.", "gold_answer": "The company missed expectations on earnings and revenue,and Elon Musk warned that the Cybertruck will not producepositive cash flow for more than a year.", "prompt": "Why did Tesla stock decline?", "usage": {"input": 469, "metric": "tokens", "output": 52, "processing_time": 1.4148402214050293, "total": 521}, "category": "complex_qa"} | |
| {"number": 177, "llm_response": "\u2022On January 5, 2021, the Democratic Party won control of the United States Senate, effective January 20, as a result of electoral victories in Georgia by Jon Ossoff in a runoff election for a six-year term and Raphael Warnock in a special runoff election for a two-year term;\r\n\u2022President-elect Biden had supported and campaigned for both candidates prior to the runoff elections on January 5", "gold_answer": "The electoral victories in Georgia by Jon Ossoff and Raphael Warnock.", "prompt": "What events enabled the Democrats to win the Senate?", "usage": {"input": 351, "metric": "tokens", "output": 101, "processing_time": 2.6644163131713867, "total": 452}, "category": "complex_qa"} | |
| {"number": 178, "llm_response": "North Industries", "gold_answer": "North Industries", "prompt": "Which company is located in Nevada?", "usage": {"input": 124, "metric": "tokens", "output": 5, "processing_time": 0.13393735885620117, "total": 129}, "category": "complex_qa"} | |
| {"number": 179, "llm_response": "Blue Moon Tech.", "gold_answer": "Blue Moon Tech.", "prompt": "Which company is located on California Street?", "usage": {"input": 125, "metric": "tokens", "output": 5, "processing_time": 0.13425731658935547, "total": 130}, "category": "complex_qa"} | |
| {"number": 180, "llm_response": "Revenue: $16.00 billion, plus or minus 2%\r\nGAAP and non-GAAP gross margins: 71.5% and 72.5%, plus or minus 50 basis points\r\nGAAP and non-GAAP operating expenses: $2.95 billion and $2.00 billion, respectively\r\nGAAP and non-GAAP other income and expense: $10", "gold_answer": "\u2022Revenue: $16.00 billion, plus or minus 2%;\n\u2022Gross Margins: 71.5%-72.%, plus or minor 50 basis points;\n\u2022GAAP operating expenses: $2.95 billion;\n\u2022Non - GAAP operating expense: $2.00 billion.", "prompt": "What are the key financial metrics for the outlook for the third quarter?", "usage": {"input": 329, "metric": "tokens", "output": 101, "processing_time": 2.643772602081299, "total": 430}, "category": "summary"} | |
| {"number": 181, "llm_response": "\u2022Second-quarter revenue was a record $10.32 billion;\r\n\u2022Second-quarter revenue was up 141% from the previous quarter;\r\n\u2022Second-quarter revenue was up 171% from the same quarter a year ago;\r\n\u2022Announced that the NVIDIA GH200 Grace Hopper Superchip for complex AI and HPC workloads is shipping this quarter;\r\n\u2022Announced that a second-generation version", "gold_answer": "\u2022Second-quarter revenue: record $10.32 billion;\n\u2022Up 141% from previous quarter;\nUp 171% from previousyear", "prompt": "What is a list of key points about data center performance?", "usage": {"input": 326, "metric": "tokens", "output": 101, "processing_time": 2.6446828842163086, "total": 427}, "category": "summary"} | |
| {"number": 182, "llm_response": "1. Record revenue of $13.51 billion, up 88% from Q1, up 101% from year ago;\r\n2. Record Data Center revenue of $10.32 billion, up 141% from Q1, up 171% from year ago;\r\n3. Non-GAAP EPS of $2.70, up 148% from Q1.", "gold_answer": "1. Record revenue of $13.51 billion;\n2. Record Data Center revenue of $10.32 billion;\n3. GAAP earnings per diluted share of $2.48.", "prompt": "What is a list of the top 3 financial highlights for the quarter?", "usage": {"input": 569, "metric": "tokens", "output": 98, "processing_time": 2.6465353965759277, "total": 667}, "category": "summary"} | |
| {"number": 183, "llm_response": "\u2022Joe Biden is the 46th president of the United States;\r\n\u2022He was inaugurated on January 20, 2021;\r\n\u2022He is a Democrat from Delaware;\r\n\u2022He previously served as vice president under Barack Obama;\r\n\u2022He won the 2020 presidential election over Donald Trump;\r\n\u2022He is the oldest president in American history.", "gold_answer": "\u2022Joe Biden was inaugurated on January 20, 2021;\n\u2022Biden is the 46th President of the United States;\n\u2022Biden previously served as Vice President under Barack Obama;\n\u2022Biden won the 2020 presidential election over incumbent Donald Trump;\n\u2022Biden is the oldest president in American history.", "prompt": "What is a list of the top summary points?", "usage": {"input": 108, "metric": "tokens", "output": 85, "processing_time": 2.165410280227661, "total": 193}, "category": "summary"} | |
| {"number": 184, "llm_response": "1. On November 23, 2020, General Services Administration head Emily W. Murphy refused to begin the transition to the president-elect, denying funds and office space to his team.\r\n2. On January 5, 2021, the Democratic Party won control of the United States Senate, effective January 20, as a result of electoral victories in Georgia.\r\n3. On January 6, 2021", "gold_answer": "1. General Services Administration head Emily W. Murphy initially refused to begin the transition for president-elect Biden until Michigan certified its results on November 23;\n2. Two days after becoming the projected winner, Biden announced the formation of a task forceto advise on the COVID-19 pandemic;\n3. On January 5, 2021, the Democratic Party won control of the U.S. Senate, with the electoralvictories in Georgia by Jon Ossoff and Raphael Warnock;\n4. On January 6, 2021, a mob of Trump supporters stormed the Capitol, forcing Congress to evacuateduring the counting of the Electoral Votes;\n5. For the inauguration, more than 26,000 National Guard members were deployed.", "prompt": "What is a list of the top 5 summary points?", "usage": {"input": 353, "metric": "tokens", "output": 101, "processing_time": 2.657489061355591, "total": 454}, "category": "summary"} | |
| {"number": 185, "llm_response": "\u2022Record revenue of $13.51 billion, up 88% from Q1, up 101% from year ago;\r\n\u2022Record Data Center revenue of $10.32 billion, up 141% from Q1, up 171% from year ago;\r\n\u2022GAAP earnings per diluted share of $2.48, up 854% from a year ago and up 202% from", "gold_answer": "\u2022Revenue: $52.9 million, up 10% in constant currency;\n\u2022Operating income: $22.4 billion, up 15% in constant currency;\n\u2022Net income: $18.3 billion, up 14% in constant currency;\n\u2022Diluted earnings per share: $2.45 billion, up 14% in constant currency.", "prompt": "What is the list of the top financial highlights for the quarter?", "usage": {"input": 567, "metric": "tokens", "output": 101, "processing_time": 2.742586135864258, "total": 668}, "category": "summary"} | |
| {"number": 186, "llm_response": "1. The study of mechanical or 'formal' reasoning began with philosophers and mathematicians in antiquity.\r\n2. The study of logic led directly to Alan Turing's theory of computation, which suggested that a machine could simulate both mathematical deduction and formal reasoning.\r\n3. The field of AI research was founded at a workshop at Dartmouth College in 1956.", "gold_answer": "1. The study of formal reasoning began in antiquity;\n2. These studies led directly to Alan Turing's theory of computation, which led researchers to consider the possibility of building an 'electronic brain';\n3. The first paper that recognized AI was published in 1943, and the field was founded in 1956.", "prompt": "What is a list of the top 3 points?", "usage": {"input": 193, "metric": "tokens", "output": 90, "processing_time": 2.3134021759033203, "total": 283}, "category": "summary"} | |
| {"number": 187, "llm_response": "\u2022The attendees became the leaders of AI research in the 1960s, and produced programs that the press described as \"astonishing\"\r\n\u2022By the mid-1960s, research was heavily funded by the Department of Defense, and laboratories had been established around the world.\r\n\u2022Herbert Simon predicted that machines would be capable of doing any work a man can do in 20 years.\r\n\u2022Marvin Minsky agreed, writing that", "gold_answer": "\u2022In the 1960s, AI researchers produced 'astonishing' programs that could learn checkers, word problems,prove logical theorems and speak English;\n\u2022AI research at the time was funded heavily by the U.S. Department of Defense;\n\u2022Herbert Simon predicted that machines would be capable in 20 years of doing human work;\n\u2022Marvin Minskey predicted that AI would be substantially solved within a generation;\n\u2022Due to these under-estimations of the difficulty of the problem, reduced government funding, and concerns about the real-world use of neural networks, the 'AI winter' followed;\n\u2022AI research was revived in the 1980s with expert systems, and the market for AI reached over $1 billion in 1985;\nHowever, with the collapse of the Lisp machine market in 1987, AI fell in disrepute again, and another AI winter ensued.", "prompt": "What is a list of the top summary points?", "usage": {"input": 415, "metric": "tokens", "output": 101, "processing_time": 2.6723103523254395, "total": 516}, "category": "summary"} | |
| {"number": 188, "llm_response": "\u2022Wednesday - $27;\r\n\u2022Thursday - $33;\r\n\u2022Friday - $36.", "gold_answer": "\u2022Wednesday - $27;\n\u2022Thursday - $33;\n\u2022Friday - $36.", "prompt": "What is a list of the stock prices on Wednesday, Thursday and Friday?", "usage": {"input": 65, "metric": "tokens", "output": 28, "processing_time": 0.7118606567382812, "total": 93}, "category": "summary"} | |
| {"number": 189, "llm_response": "Joe Biden, 46th President, inaugurated on January 20, 2021, after winning 2020 election.", "gold_answer": "Joe Biden is the 46th President of the United States.", "prompt": "What is a headline summary in 10 words or less?", "usage": {"input": 112, "metric": "tokens", "output": 34, "processing_time": 0.8711633682250977, "total": 146}, "category": "summary"} | |
| {"number": 190, "llm_response": "On January 6, 2021, a mob of Trump supporters violently stormed the Capitol in an attempt to overturn the election, forcing Congress to evacuate.", "gold_answer": "After Biden won the election, he formed a pandemic task force, the democrats won the Senate, and a mob stormed the Capital on January 6.", "prompt": "What is a one-line brief description in 25 words or less?", "usage": {"input": 357, "metric": "tokens", "output": 39, "processing_time": 1.0567092895507812, "total": 396}, "category": "summary"} | |
| {"number": 191, "llm_response": "Not Found.", "gold_answer": "\u2022Office Commercial Products;\n\u2022Office Consumer Products;\n\u2022Microsoft 365 Consumer;\n\u2022LinkedIn;\n\u2022Dynamics Products;\n\u2022Server Products;\n\u2022More Personal Computing;\n\u2022Windows OEM;\n\u2022Devices;\n\u2022Windows Commercial;\n\u2022Xbox;\n\u2022Search and News Advertising.", "prompt": "What is a list of top Microsoft product offerings?", "usage": {"input": 325, "metric": "tokens", "output": 4, "processing_time": 0.15287423133850098, "total": 329}, "category": "summary"} | |
| {"number": 192, "llm_response": "\u2022Invoice Date: March 3, 2022\r\n\u2022Invoice No: 234\r\n\u2022ABC Company 8675 River Run Road Marlington, CT 09392\r\n\u2022Invoice To: Delta Inc.\r\n8723 Wilton Boulevard, Los Angeles, CA\r\n\u2022Payment Terms DUE DATE: May 3, 2022\r\n\u2022Quantity Description", "gold_answer": "\u2022Invoice Date: March 3, 2022;\\\u2022Invoice Number: No 234;\n\u2022From: ABC Company;\n\u2022To: Delta Inc.;\n\u2022Due Date: Due on Receipt;\n\u2022Total: $4565.00.", "prompt": "What is a list of the key points?", "usage": {"input": 222, "metric": "tokens", "output": 101, "processing_time": 2.6166954040527344, "total": 323}, "category": "summary"} | |
| {"number": 193, "llm_response": "Microsoft announced that its cloud revenue was up 22% year-over-year, to $28.5 billion, and that the world's most advanced AI models are coming together with the world's most universal user interface - natural language - to create a new era of computing.", "gold_answer": "According to Microsoft's CEO and CFO, the Microsoft Cloud is the platform for choice for AI,and growing 22% year-over-year as a result of focused execution.", "prompt": "What is a summary in 30 words or less?", "usage": {"input": 167, "metric": "tokens", "output": 61, "processing_time": 1.5699002742767334, "total": 228}, "category": "summary"} | |
| {"number": 194, "llm_response": "\u2022Vivek Murthy,\r\n\u2022David A. Kessler,\r\n\u2022Marcella Nunez-Smith,\r\n\u2022Jon Ossoff,\r\n\u2022Raphael Warnock,\r\n\u2022Emily W. Murphy, and\r\n\u2022General Services Administration.", "gold_answer": "\u2022Biden;\n\u2022Emily W. Murphy;\n\u2022Vivek Murthy;\n\u2022David A. Kessler;\n\u2022Marcella Nunez-Smith;\n\u2022Jon Ossoff;\n\u2022Raphael Warnock.", "prompt": "What is a list of all of the people mentioned?", "usage": {"input": 352, "metric": "tokens", "output": 62, "processing_time": 1.6528639793395996, "total": 414}, "category": "summary"} | |
| {"number": 195, "llm_response": "Nokia is cutting up to 14,000 jobs and reducing costs in the face of a challenging market environment. Nokia reported a 20% decline in net sales and a 69% decline in profit.", "gold_answer": "Nokia is cutting 14,000 jobs as part of a cost reductionplan after reporting profits that declined 69% year-on-year.", "prompt": "What is a one-line summary?", "usage": {"input": 140, "metric": "tokens", "output": 52, "processing_time": 1.337764024734497, "total": 192}, "category": "summary"} | |
| {"number": 196, "llm_response": "\u2022Senior Vice President, Event Planning (\"SVP\") of the Employer's Workforce Optimization Division;\r\n\u2022Reporting to the Board of Directors;\r\n\u2022Devoting substantially all of his business time and attention;\r\n\u2022Responsible for running day-to-day operations;\r\n\u2022Responsible for performing faithfully, diligently and competently the duties and responsibilities of a SVP.", "gold_answer": "\u2022Senior Vice President, Event Planning ('SVP') of Workforce Optimization Division;\n\u2022Report to Board of Directors;\n\u2022Devote substantially all of business time and attention to the operations of the Employer;\n\u2022responsible for running Employer's day-to-day operations;\n\u2022Perform faithfully, diligently and competently the duties and responsibilities of a SVP.", "prompt": "What is a list of the executive's responsibilities?", "usage": {"input": 384, "metric": "tokens", "output": 92, "processing_time": 2.4456288814544678, "total": 476}, "category": "summary"} | |
| {"number": 197, "llm_response": "Nvidia Announces Financial Results for Second Quarter Fiscal 2024\r\n\r\n\u2022Record revenue of $13.51 billion, up 88% from Q1, up 101% from year ago\r\n\r\n\u2022Record Data Center revenue of $10.32 billion, up 141% from Q1, up 171% from year ago", "gold_answer": "NVIDIA Announces Second Quarter Fiscal 2024 Results with record revenue of $13.51 billion.", "prompt": "What is a headline description?", "usage": {"input": 561, "metric": "tokens", "output": 91, "processing_time": 2.475609540939331, "total": 652}, "category": "summary"} | |
| {"number": 198, "llm_response": "Microsoft CEO Satya Nadella: \"The world's most advanced AI models are coming together with the world's most universal user interface - natural language - to create a new era of computing.\"", "gold_answer": "Microsoft Cloud is the platform of choice for the next generation of AI.", "prompt": "What is a summary of the CEO's statement in 15 words or less?", "usage": {"input": 173, "metric": "tokens", "output": 42, "processing_time": 1.0853066444396973, "total": 215}, "category": "summary"} | |
| {"number": 199, "llm_response": "\u2022Services Vendor Inc. 100 Elm Street, Pleasantville, NY 5900 1st Street, Los Angeles, CA \r\n\u2022Description - Front End Engineering Service - $5000.00\r\n\u2022Back End Engineering Service - $7500.00\r\n\u2022Quality Assurance Manager - $10,000.00\r\n\u2022Total Amount - $22,500", "gold_answer": "\u2022Invoice From: Services Vendor Inc.;\n\u2022Invoice To: Alpha Inc.;\n\u2022Total Amount: $22,500.00;\n\u2022Payment Due: within 30 days;\n\u2022Contact: Bia Hermes;\n\u2022Invoice #0001;\u2022Data: 01/01/2022.", "prompt": "What are the key terms of the invoice?", "usage": {"input": 203, "metric": "tokens", "output": 101, "processing_time": 2.6057426929473877, "total": 304}, "category": "summary"} | |