Upload news summarization eval results for 69d59c104adb81dd2de75563

eval/news_summarization/results/jobs/69d59c104adb81dd2de75563/results.json

@@ -0,0 +1,219 @@
+{
+  "summary": {
+    "provider": "hf-transformers",
+    "model": "google/gemma-3-12b-it",
+    "dataset": "bbc2024_qwen_reference",
+    "prompt_style": "simple",
+    "cases": 10,
+    "average_token_f1": 0.4922,
+    "average_rouge_l_f1_approx": 0.3793,
+    "average_word_count": 55.2,
+    "median_latency_ms": 34305.84,
+    "average_rouge1_f1": 0.5295,
+    "average_rouge2_f1": 0.2673,
+    "average_rougeL_f1": 0.3921,
+    "average_bertscore_f1": 0.9167
+  },
+  "rows": [
+    {
+      "case_id": "news-001",
+      "article_chars": 4019,
+      "reference_summary": "West Ham captain Dagny Brynjarsdottir has been praised for a documentary that showcases her pregnancy journey as an elite footballer, with the Iceland midfielder's film 'Omarsson' highlighting the challenges and joys of balancing motherhood and sport. The documentary, released by West Ham, is being seen as a milestone for female athletes, with Brynjarsdottir's story inspiring others and demonstrating the support now available for players starting families.",
+      "prediction": "West Ham captain Dagny Brynjarsdottir is featured in a new documentary, \"Omarsson,\" showcasing her pregnancy journey and the support she received from her club while continuing her career as an elite footballer. The film highlights the changing landscape of professional sports, with increased support for athletes balancing motherhood, inspiring others like Katrina Gorry and Emma Harries.",
+      "source_url": null,
+      "latency_ms": 35545.39,
+      "scores": {
+        "token_f1": 0.5082,
+        "rouge_l_f1_approx": 0.377,
+        "rouge1_f1": 0.5484,
+        "rouge2_f1": 0.2295,
+        "rougeL_f1": 0.4194,
+        "word_count": 56,
+        "bertscore_f1": 0.9108
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-002",
+      "article_chars": 2779,
+      "reference_summary": "US authorities have charged Takeshi Ebisawa, alleged to be a senior member of the Japanese Yakuza crime syndicate, with conspiring to traffic nuclear materials, attempting to sell uranium and plutonium to what he believed were Iranian agents for a nuclear bomb. Ebisawa and a Thai co-defendant, Somphop Singhasiri, face life imprisonment if convicted, with Ebisawa also accused of seeking to acquire military-grade weapons for a Myanmar rebel group. The nuclear materials, sourced from Myanmar, were seized in Thailand and confirmed to contain uranium and weapons-grade plutonium by a US laboratory.",
+      "prediction": "A senior Yakuza figure, Takeshi Ebisawa, has been charged in the US with conspiring to traffic nuclear materials to Iran, allegedly offering uranium and plutonium to an undercover agent. He also faces charges related to weapons trafficking and money laundering, alongside a Thai co-defendant, with authorities seizing nuclear samples and uncovering plans to acquire military-grade weapons.",
+      "source_url": null,
+      "latency_ms": 34398.38,
+      "scores": {
+        "token_f1": 0.4795,
+        "rouge_l_f1_approx": 0.3288,
+        "rouge1_f1": 0.543,
+        "rouge2_f1": 0.2282,
+        "rougeL_f1": 0.3444,
+        "word_count": 56,
+        "bertscore_f1": 0.9077
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-003",
+      "article_chars": 6612,
+      "reference_summary": "Heavy fighting has erupted around al-Amal Hospital in Khan Younis, Gaza, as Israeli ground forces encircle the city, with reports of troops advancing and intense clashes near medical facilities. At least 195 Palestinians and 24 Israeli soldiers have been killed in the recent escalation, with the Israeli military targeting Hamas leaders believed to be hiding in tunnels. The World Health Organization expresses concern over attacks on hospitals, while residents face difficulties fleeing the area.",
+      "prediction": "Intense fighting continues in Khan Younis, southern Gaza, as Israeli forces encircle the city and advance into Hamas strongholds, leading to civilian displacement and concerns over the safety of hospitals. Amidst rising casualties on both sides, negotiations for a ceasefire are ongoing, but a deal remains elusive as Hamas seeks a permanent cessation of hostilities.",
+      "source_url": null,
+      "latency_ms": 33228.81,
+      "scores": {
+        "token_f1": 0.3256,
+        "rouge_l_f1_approx": 0.2481,
+        "rouge1_f1": 0.3538,
+        "rouge2_f1": 0.125,
+        "rougeL_f1": 0.2615,
+        "word_count": 55,
+        "bertscore_f1": 0.8949
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-004",
+      "article_chars": 3452,
+      "reference_summary": "Simon Rimmer's vegetarian restaurant, Greens in Manchester, has closed after 33 years due to a rise in rent and operating costs, making the business unviable. The TV chef expressed his heartbreak over the closure on social media, thanking staff, customers, and suppliers.",
+      "prediction": "The vegetarian restaurant Greens, co-owned by TV chef Simon Rimmer, has closed after 33 years due to rising rent and operating costs. Rimmer announced the immediate closure on social media, expressing heartbreak and thanking staff, customers, and suppliers, while noting that its sister restaurant in Sale remains open.",
+      "source_url": null,
+      "latency_ms": 30203.54,
+      "scores": {
+        "token_f1": 0.6667,
+        "rouge_l_f1_approx": 0.5333,
+        "rouge1_f1": 0.7174,
+        "rouge2_f1": 0.4444,
+        "rougeL_f1": 0.5435,
+        "word_count": 48,
+        "bertscore_f1": 0.9438
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-005",
+      "article_chars": 3581,
+      "reference_summary": "NASA's Artemis III mission, which aims to return astronauts to the Moon, has been delayed until September 2026 due to the need for further technology development, including a landing craft and updated spacesuits. The precursor Artemis II mission, originally planned for the end of 2022, is now scheduled for September 2025. SpaceX's Starship vehicle is set to be the human landing system for Artemis III, requiring extensive testing and demonstrations before it can be used for a lunar landing.",
+      "prediction": "NASA has announced delays for the Artemis program, pushing the return of humans to the Moon to September 2026 at the earliest and a fly-around-the-Moon mission to September 2025, citing the need for further technology development and safety considerations. The delays are primarily due to challenges with the lunar landing craft being developed by SpaceX, as well as concerns about the Orion capsule's life-support system and heat shield.",
+      "source_url": null,
+      "latency_ms": 43632.87,
+      "scores": {
+        "token_f1": 0.449,
+        "rouge_l_f1_approx": 0.2857,
+        "rouge1_f1": 0.5065,
+        "rouge2_f1": 0.1711,
+        "rougeL_f1": 0.2857,
+        "word_count": 68,
+        "bertscore_f1": 0.8979
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-006",
+      "article_chars": 5882,
+      "reference_summary": "Multiple news agencies, including PA, Getty Images, AFP, Reuters, and Associated Press, have withdrawn a photo of the Princess of Wales and her children due to concerns it may have been digitally manipulated. The image, taken by Prince William for Mother's Day and the first official photo of the princess since her surgery, was initially posted by Kensington Palace on social media. The agencies are seeking clarification from the palace, and in the absence of it, have removed the photo from their services.",
+      "prediction": "Major news agencies have retracted a Mother's Day photo of the Princess of Wales and her children due to concerns about digital manipulation and inconsistencies within the image. The photo, released by Kensington Palace, was intended to reassure the public about her recovery following abdominal surgery but instead fueled further speculation.",
+      "source_url": null,
+      "latency_ms": 27817.1,
+      "scores": {
+        "token_f1": 0.4478,
+        "rouge_l_f1_approx": 0.3582,
+        "rouge1_f1": 0.4853,
+        "rouge2_f1": 0.2836,
+        "rougeL_f1": 0.3824,
+        "word_count": 51,
+        "bertscore_f1": 0.9142
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-007",
+      "article_chars": 4085,
+      "reference_summary": "The death of Dragon Ball creator Akira Toriyama has been mourned by fans worldwide, with memories shared of a 2018 diplomatic incident between Japan and Mexico over unauthorized public screenings of the series. Despite copyright issues, a free screening of Dragon Ball Super's episode #130 was eventually allowed in Ciudad Juarez, attracting thousands of fans.",
+      "prediction": "Following the death of Dragon Ball creator Akira Toriyama, fans worldwide are celebrating his legacy, recalling how the series sparked a diplomatic incident in Mexico six years ago over unauthorized public screenings. Despite initial copyright concerns and intervention from the Japanese Embassy, a screening in Ciudad Juarez was ultimately approved, drawing an estimated 15,000 attendees.",
+      "source_url": null,
+      "latency_ms": 34305.84,
+      "scores": {
+        "token_f1": 0.5273,
+        "rouge_l_f1_approx": 0.4545,
+        "rouge1_f1": 0.5179,
+        "rouge2_f1": 0.2727,
+        "rougeL_f1": 0.4464,
+        "word_count": 55,
+        "bertscore_f1": 0.9387
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-008",
+      "article_chars": 3631,
+      "reference_summary": "North Korea fired over 200 artillery shells off its west coast towards South Korea's Yeonpyeong island, prompting the South to hold live fire drills and order civilians to seek shelter; no damage was reported, but the South called it a provocative act, while the North claimed it was a response to military actions by the South. This tension follows North Korea's suspension of a military deal with the South and its recent satellite launch, which led to heightened tensions along the border.",
+      "prediction": "North Korea fired over 200 artillery shells towards South Korea's Yeonpyeong island, prompting South Korea to conduct live fire drills and issue shelter warnings to nearby residents. This exchange of fire, following North Korea's withdrawal from a military agreement and heightened tensions, raises concerns about peace and stability on the Korean peninsula.",
+      "source_url": null,
+      "latency_ms": 33145.14,
+      "scores": {
+        "token_f1": 0.4627,
+        "rouge_l_f1_approx": 0.4478,
+        "rouge1_f1": 0.4928,
+        "rouge2_f1": 0.2941,
+        "rougeL_f1": 0.4783,
+        "word_count": 52,
+        "bertscore_f1": 0.9288
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-009",
+      "article_chars": 4821,
+      "reference_summary": "Trainer Willie Mullins achieved a historic 100th Cheltenham Festival victory as his son Patrick won the Champion Bumper on Jasmin De Vaux, becoming the first person to reach a century of wins at the event. Mullins, with six victories this year, celebrated the milestone with a hat doff, and his success is a testament to his dedication, team, and innovative approach, outpacing second-placed Nicky Henderson's 73 wins.",
+      "prediction": "Willie Mullins achieved a historic 100th victory at the Cheltenham Festival, solidifying his dominance in the racing world and extending his lead over rivals like Nicky Henderson. The milestone win, secured by his son Patrick in the Champion Bumper, caps a remarkable career marked by innovation, a strong team, and resilience in the face of setbacks.",
+      "source_url": null,
+      "latency_ms": 34445.15,
+      "scores": {
+        "token_f1": 0.4715,
+        "rouge_l_f1_approx": 0.2927,
+        "rouge1_f1": 0.512,
+        "rouge2_f1": 0.2276,
+        "rougeL_f1": 0.288,
+        "word_count": 56,
+        "bertscore_f1": 0.9019
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    },
+    {
+      "case_id": "news-010",
+      "article_chars": 2448,
+      "reference_summary": "Cambridge University Library has acquired an archive of posters and memorabilia from the Poems On The Underground project, which since 1986 has featured literary works on London's Tube network, including a letter from Philip Larkin who never saw his poetry displayed. The archive, now open to the public, also contains a note from Seamus Heaney praising the project's impact on poetry's presence in public spaces.",
+      "prediction": "Hundreds of posters and memorabilia from the \"Poems On The Underground\" public art project have been donated to Cambridge University Library, including a letter from Philip Larkin who never saw his work displayed. The archive, which also features correspondence from Seamus Heaney and details readings and publications, is now publicly accessible for research and study.",
+      "source_url": null,
+      "latency_ms": 31064.33,
+      "scores": {
+        "token_f1": 0.5833,
+        "rouge_l_f1_approx": 0.4667,
+        "rouge1_f1": 0.6179,
+        "rouge2_f1": 0.3967,
+        "rougeL_f1": 0.4715,
+        "word_count": 55,
+        "bertscore_f1": 0.9284
+      },
+      "provider_metadata": {
+        "model": "google/gemma-3-12b-it"
+      }
+    }
+  ]
+}