Upload

app.py

@@ -0,0 +1,135 @@
+import gradio as gr
+from src.styles import custom_css
+
+from src.structures.all_structure import (
+    ALL_ACCS,
+    STYLED
+    # ALL_DATA_TYPES,
+    # ALL_ORDER_LIST
+)
+from src.structures.pes_structure import (PES_ACCS,
+    ORDER_LIST,
+    DATA_TYPES,
+    COLUMN_HEADERS,
+    filter_data,
+    filter_columns,
+)
+from src.abouts import *
+from src.structures.lek_structure import (
+    LEK_ACCS,
+    ORDER_LIST_LEK,
+    COLUMN_HEADERS_LEK,
+    DATA_TYPES_LEK,
+    filter_columns_lek
+)
+from src.structures.ldek_structure import (
+    LDEK_ACCS,
+    ORDER_LIST_LDEK,
+    COLUMN_HEADERS_LDEK,
+    DATA_TYPES_LDEK,
+    filter_columns_ldek
+)
+
+global data_component
+global data_component_ldek
+
+main = gr.Blocks(css=custom_css)
+with main:
+    with gr.Row():
+        with gr.Column():
+            image = gr.Image("src/images/logo.png",
+                show_download_button=False,
+                show_share_button=False,
+                show_fullscreen_button=False,
+                container=False)
+        with gr.Column():
+            gr.HTML(HEADER_TITLE)
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+        with gr.TabItem("🥇 Medical leaderboard"):
+            gr.Markdown(LEADERBOARD_DESC)
+            data_component = gr.components.Dataframe(
+                value=STYLED,
+                type="pandas",
+                datatype=["markdown"],
+                interactive=False,
+                visible=True,
+                show_label=True,
+                column_widths=[400,150,150,150,150]
+            )
+
+        with gr.TabItem("🏥 PES"):
+            gr.Markdown(PES_DESC)
+            # Checkbox to toggle column visibility
+            columns_selector = gr.CheckboxGroup(
+                choices=ORDER_LIST,
+                label="Select columns to display",
+                value=ORDER_LIST,
+            )
+            # Dataframe component to display the leaderboard data
+            data_component = gr.components.Dataframe(
+                value=PES_ACCS,
+                headers=COLUMN_HEADERS,
+                type="pandas",
+                datatype=DATA_TYPES,
+                interactive=False,
+                visible=True,
+                # column_widths=[400] + [250] * (len(COLUMN_HEADERS) - 1)
+            )
+            def update_dataframe(selected_columns):
+                return filter_columns(selected_columns)
+            columns_selector.change(update_dataframe, inputs=columns_selector, outputs=data_component)
+
+        with gr.TabItem("🦷 LDEK"):
+            gr.Markdown(LDEK_DESC)
+            columns_selector = gr.CheckboxGroup(
+                choices=ORDER_LIST_LDEK,
+                label="Select columns to display",
+                value=ORDER_LIST_LDEK,
+            )
+            data_component_ldek = gr.components.Dataframe(
+                value=LDEK_ACCS,
+                headers=COLUMN_HEADERS_LDEK,
+                type="pandas",
+                datatype=DATA_TYPES_LDEK,
+                interactive=False,
+                visible=True,
+                column_widths=[400] + [155] * 23
+            )
+            def update_dataframe(selected_columns):
+                return filter_columns_ldek(selected_columns)
+            columns_selector.change(update_dataframe, inputs=columns_selector, outputs=data_component_ldek)
+
+
+        with gr.TabItem("🩺 LEK"):
+            gr.Markdown(LEK_DESC)
+            columns_selector = gr.CheckboxGroup(
+                choices=ORDER_LIST_LEK,
+                label="Select columns to display",
+                value=ORDER_LIST_LEK,
+            )
+            data_component_ldek = gr.components.Dataframe(
+                value=LEK_ACCS,
+                headers=COLUMN_HEADERS_LEK,
+                type="pandas",
+                datatype=DATA_TYPES_LEK,
+                interactive=False,
+                visible=True,
+                column_widths=[400] + [155] * 23
+            )
+            def update_dataframe(selected_columns):
+                return filter_columns_lek(selected_columns)
+            columns_selector.change(update_dataframe, inputs=columns_selector, outputs=data_component_ldek)
+
+
+    with gr.Column():
+            with gr.Accordion("📙 Citation", open=False):
+                citation_button = gr.Textbox(
+                    label=CITATION_LABEL,
+                    value=CITATION_CONTENT,
+                    lines=20,
+                    elem_id="citation-button",
+                    show_copy_button=True,
+                )
+
+if __name__ == "__main__":
+    main.launch()

leaderboards/pes_accuracy.json

@@ -0,0 +1,266 @@
+[
+    {
+        "model_name": "model-epfl-llm-meditron-7b",
+        "stomatologia dziecięca": 15.833333333333332,
+        "neurologia": 14.285714285714285,
+        "medycyna pracy": 17.796610169491526,
+        "medycyna rodzinna": 20.689655172413794,
+        "patomorfologia": 25.0,
+        "endokrynologia ginekologiczna i rozrodczość": 19.491525423728813,
+        "radiologia i diagnostyka obrazowa": 18.333333333333332,
+        "chirurgia szczękowo-twarzowa": 26.05042016806723,
+        "medycyna sportowa": 25.0,
+        "neonatologia": 25.423728813559322,
+        "ortopedia": 18.26086956521739,
+        "medycyna paliatywna": 20.0,
+        "chirurgia stomatologiczna": 16.94915254237288,
+        "neurologia dziecięca": 26.05042016806723,
+        "dermatologia i wenerologia": 15.966386554621847,
+        "diabetologia": 19.65811965811966,
+        "hematologia": 21.008403361344538,
+        "onkologia kliniczna": 21.367521367521366,
+        "protetyka stomatologiczna": 20.0,
+        "endokrynologia i diabetologia dziecięca": 21.008403361344538,
+        "neurochirurgia": 23.728813559322035,
+        "alergologia": 20.833333333333336,
+        "geriatria": 16.101694915254235,
+        "choroby wewnętrzne": 17.647058823529413,
+        "ginekologia onkologiczna": 13.445378151260504,
+        "otolaryngologia": 27.350427350427353,
+        "ortodoncja": 25.0,
+        "anestezjologia": 24.369747899159663,
+        "endokrynologia": 17.5,
+        "periodontologia": 11.76470588235294,
+        "psychiatria dzieci i mlodzieży": 19.166666666666668,
+        "psychiatria": 18.333333333333332,
+        "rehabilitacja medyczna": 16.94915254237288,
+        "perinatologia": 8.333333333333332,
+        "balneologia i medycyna fizykalna": 23.333333333333332,
+        "pediatria": 10.784313725490197,
+        "nefrologia": 22.5,
+        "choroby płuc": 18.103448275862068,
+        "choroby zakaźne": 20.0,
+        "okulistyka": 17.094017094017094,
+        "kardiochirurgia": 14.166666666666666,
+        "chirurgia naczyniowa": 19.166666666666668,
+        "reumatologia": 18.96551724137931,
+        "chirurgia onkologiczna": 12.711864406779661,
+        "transplantologia kliniczna": 18.803418803418804,
+        "radioterapia onkologiczna": 26.666666666666668,
+        "gastroenterologia dziecięca": 14.166666666666666,
+        "kardiologia": 17.094017094017094,
+        "angiologia": 14.166666666666666,
+        "gastroenterologia": 17.543859649122805,
+        "chirurgia dziecięca": 16.80672268907563,
+        "hipertensjologia": 10.92436974789916,
+        "stomatologia zachowawcza": 26.05042016806723,
+        "chirurgia ogólna": 19.166666666666668,
+        "choroby płuc dzieci": 24.166666666666668,
+        "położnictwo i ginekologia": 21.666666666666668,
+        "medycyna ratunkowa": 19.626168224299064,
+        "nefrologia dziecięca": 20.168067226890756,
+        "endokrynologia ginekologiczna i rozrodczość": 24.786324786324787,
+        "anestezjologia i intensywna terapia": 23.47826086956522,
+        "stomatologia dziecięca": 23.52941176470588,
+        "ortopedia i traumatologia narządu ruchu": 21.428571428571427,
+        "położnictwo i ginekologia": 15.65217391304348,
+        "chirurgia szczękowo-twarzowa": 18.421052631578945,
+        "neurologia dziecięca": 23.52941176470588,
+        "choroby zakaźne": 18.333333333333332,
+        "gastroenterologia dziecięca": 14.166666666666666,
+        "stomatologia zachowawcza z endodoncją": 19.491525423728813,
+        "endokrynologia i diabetologia dziecięca": 26.27118644067797,
+        "psychiatria dzieci i młodzieży": 16.94915254237288,
+        "chirurgia ogólna": 20.33898305084746,
+        "choroby wewnętrzne": 18.556701030927837,
+        "diagnostyka laboratoryjna": 30.508474576271187,
+        "chirurgia dziecięca": 22.88135593220339,
+        "otorynolaryngologia": 18.803418803418804,
+        "urologia dziecięca": 25.663716814159294,
+        "zdrowie publiczne": 20.168067226890756,
+        "genetyka kliniczna": 19.491525423728813,
+        "medycyna nuklearna": 23.076923076923077,
+        "kardiologia dziecięca": 15.789473684210526,
+        "chirurgia plastyczna": 20.0,
+        "chirurgia klatki piersiowej": 24.166666666666668,
+        "audiologia i foniatria": 18.421052631578945,
+        "intensywna terapia": 20.833333333333336,
+        "overall_accuracy": 19.74083822636161
+    },
+    {
+        "model_name": "model-speakleash-Bielik-11B-v2.2-Instruct",
+        "stomatologia dziecięca": 40.833333333333336,
+        "neurologia": 48.739495798319325,
+        "medycyna pracy": 42.3728813559322,
+        "medycyna rodzinna": 43.103448275862064,
+        "patomorfologia": 50.83333333333333,
+        "endokrynologia ginekologiczna i rozrodczość": 32.20338983050847,
+        "radiologia i diagnostyka obrazowa": 46.666666666666664,
+        "chirurgia szczękowo-twarzowa": 25.210084033613445,
+        "medycyna sportowa": 57.49999999999999,
+        "neonatologia": 46.61016949152542,
+        "ortopedia": 41.73913043478261,
+        "medycyna paliatywna": 45.83333333333333,
+        "chirurgia stomatologiczna": 36.440677966101696,
+        "neurologia dziecięca": 45.378151260504204,
+        "dermatologia i wenerologia": 37.81512605042017,
+        "diabetologia": 31.62393162393162,
+        "hematologia": 33.61344537815126,
+        "onkologia kliniczna": 44.44444444444444,
+        "protetyka stomatologiczna": 36.666666666666664,
+        "endokrynologia i diabetologia dziecięca": 44.537815126050425,
+        "neurochirurgia": 27.966101694915253,
+        "alergologia": 36.666666666666664,
+        "geriatria": 45.76271186440678,
+        "choroby wewnętrzne": 46.21848739495798,
+        "ginekologia onkologiczna": 42.016806722689076,
+        "otolaryngologia": 33.33333333333333,
+        "ortodoncja": 31.25,
+        "anestezjologia": 51.26050420168067,
+        "endokrynologia": 33.33333333333333,
+        "periodontologia": 36.97478991596639,
+        "psychiatria dzieci i mlodzieży": 55.833333333333336,
+        "psychiatria": 36.666666666666664,
+        "rehabilitacja medyczna": 53.38983050847458,
+        "perinatologia": 34.166666666666664,
+        "balneologia i medycyna fizykalna": 41.66666666666667,
+        "pediatria": 48.03921568627451,
+        "nefrologia": 33.33333333333333,
+        "choroby płuc": 51.724137931034484,
+        "choroby zakaźne": 45.0,
+        "okulistyka": 45.2991452991453,
+        "kardiochirurgia": 37.5,
+        "chirurgia naczyniowa": 45.83333333333333,
+        "reumatologia": 36.206896551724135,
+        "chirurgia onkologiczna": 44.06779661016949,
+        "transplantologia kliniczna": 33.33333333333333,
+        "radioterapia onkologiczna": 41.66666666666667,
+        "gastroenterologia dziecięca": 39.166666666666664,
+        "kardiologia": 40.17094017094017,
+        "angiologia": 46.666666666666664,
+        "gastroenterologia": 45.614035087719294,
+        "chirurgia dziecięca": 42.857142857142854,
+        "hipertensjologia": 51.26050420168067,
+        "stomatologia zachowawcza": 27.73109243697479,
+        "chirurgia ogólna": 29.166666666666668,
+        "choroby płuc dzieci": 48.333333333333336,
+        "położnictwo i ginekologia": 40.833333333333336,
+        "medycyna ratunkowa": 50.467289719626166,
+        "nefrologia dziecięca": 50.42016806722689,
+        "endokrynologia ginekologiczna i rozrodczość": 41.02564102564102,
+        "anestezjologia i intensywna terapia": 39.130434782608695,
+        "stomatologia dziecięca": 32.773109243697476,
+        "ortopedia i traumatologia narządu ruchu": 40.17857142857143,
+        "położnictwo i ginekologia": 42.608695652173914,
+        "chirurgia szczękowo-twarzowa": 27.192982456140353,
+        "neurologia dziecięca": 42.016806722689076,
+        "choroby zakaźne": 40.0,
+        "gastroenterologia dziecięca": 42.5,
+        "stomatologia zachowawcza z endodoncją": 36.440677966101696,
+        "endokrynologia i diabetologia dziecięca": 33.89830508474576,
+        "psychiatria dzieci i młodzieży": 59.32203389830508,
+        "chirurgia ogólna": 46.61016949152542,
+        "choroby wewnętrzne": 38.144329896907216,
+        "diagnostyka laboratoryjna": 50.0,
+        "chirurgia dziecięca": 29.66101694915254,
+        "otorynolaryngologia": 41.88034188034188,
+        "urologia dziecięca": 35.39823008849557,
+        "zdrowie publiczne": 71.42857142857143,
+        "genetyka kliniczna": 38.13559322033898,
+        "medycyna nuklearna": 45.2991452991453,
+        "kardiologia dziecięca": 42.98245614035088,
+        "chirurgia plastyczna": 45.83333333333333,
+        "chirurgia klatki piersiowej": 39.166666666666664,
+        "audiologia i foniatria": 48.24561403508772,
+        "intensywna terapia": 44.166666666666664,
+        "overall_accuracy": 41.759465478841875
+    },
+    {
+        "model_name": "model-meta-llama-Meta-Llama-3.1-8B-Instruct",
+        "stomatologia dziecięca": 27.500000000000004,
+        "neurologia": 50.42016806722689,
+        "medycyna pracy": 38.983050847457626,
+        "medycyna rodzinna": 35.3448275862069,
+        "patomorfologia": 46.666666666666664,
+        "endokrynologia ginekologiczna i rozrodczość": 27.11864406779661,
+        "radiologia i diagnostyka obrazowa": 40.833333333333336,
+        "chirurgia szczękowo-twarzowa": 26.89075630252101,
+        "medycyna sportowa": 45.0,
+        "neonatologia": 45.76271186440678,
+        "ortopedia": 40.0,
+        "medycyna paliatywna": 55.833333333333336,
+        "chirurgia stomatologiczna": 33.05084745762712,
+        "neurologia dziecięca": 38.655462184873954,
+        "dermatologia i wenerologia": 32.773109243697476,
+        "diabetologia": 41.88034188034188,
+        "hematologia": 41.17647058823529,
+        "onkologia kliniczna": 36.75213675213676,
+        "protetyka stomatologiczna": 38.333333333333336,
+        "endokrynologia i diabetologia dziecięca": 41.17647058823529,
+        "neurochirurgia": 27.966101694915253,
+        "alergologia": 37.5,
+        "geriatria": 44.91525423728814,
+        "choroby wewnętrzne": 47.05882352941176,
+        "ginekologia onkologiczna": 40.33613445378151,
+        "otolaryngologia": 26.495726495726498,
+        "ortodoncja": 27.67857142857143,
+        "anestezjologia": 48.739495798319325,
+        "endokrynologia": 30.833333333333336,
+        "periodontologia": 41.17647058823529,
+        "psychiatria dzieci i mlodzieży": 47.5,
+        "psychiatria": 32.5,
+        "rehabilitacja medyczna": 45.76271186440678,
+        "perinatologia": 36.666666666666664,
+        "balneologia i medycyna fizykalna": 38.333333333333336,
+        "pediatria": 39.21568627450981,
+        "nefrologia": 37.5,
+        "choroby płuc": 50.86206896551724,
+        "choroby zakaźne": 44.166666666666664,
+        "okulistyka": 43.58974358974359,
+        "kardiochirurgia": 43.333333333333336,
+        "chirurgia naczyniowa": 35.833333333333336,
+        "reumatologia": 36.206896551724135,
+        "chirurgia onkologiczna": 37.28813559322034,
+        "transplantologia kliniczna": 32.47863247863248,
+        "radioterapia onkologiczna": 35.833333333333336,
+        "gastroenterologia dziecięca": 40.833333333333336,
+        "kardiologia": 37.60683760683761,
+        "angiologia": 40.833333333333336,
+        "gastroenterologia": 39.473684210526315,
+        "chirurgia dziecięca": 31.092436974789916,
+        "hipertensjologia": 46.21848739495798,
+        "stomatologia zachowawcza": 37.81512605042017,
+        "chirurgia ogólna": 25.0,
+        "choroby płuc dzieci": 37.5,
+        "położnictwo i ginekologia": 36.666666666666664,
+        "medycyna ratunkowa": 50.467289719626166,
+        "nefrologia dziecięca": 43.69747899159664,
+        "endokrynologia ginekologiczna i rozrodczość": 42.73504273504273,
+        "anestezjologia i intensywna terapia": 36.52173913043478,
+        "stomatologia dziecięca": 31.092436974789916,
+        "ortopedia i traumatologia narządu ruchu": 38.392857142857146,
+        "położnictwo i ginekologia": 39.130434782608695,
+        "chirurgia szczękowo-twarzowa": 26.31578947368421,
+        "neurologia dziecięca": 41.17647058823529,
+        "choroby zakaźne": 30.0,
+        "gastroenterologia dziecięca": 38.333333333333336,
+        "stomatologia zachowawcza z endodoncją": 27.11864406779661,
+        "endokrynologia i diabetologia dziecięca": 38.983050847457626,
+        "psychiatria dzieci i młodzieży": 51.69491525423729,
+        "chirurgia ogólna": 41.52542372881356,
+        "choroby wewnętrzne": 43.29896907216495,
+        "diagnostyka laboratoryjna": 48.30508474576271,
+        "chirurgia dziecięca": 33.05084745762712,
+        "otorynolaryngologia": 36.75213675213676,
+        "urologia dziecięca": 35.39823008849557,
+        "zdrowie publiczne": 61.34453781512605,
+        "genetyka kliniczna": 46.61016949152542,
+        "medycyna nuklearna": 43.58974358974359,
+        "kardiologia dziecięca": 42.98245614035088,
+        "chirurgia plastyczna": 45.0,
+        "chirurgia klatki piersiowej": 35.833333333333336,
+        "audiologia i foniatria": 47.368421052631575,
+        "intensywna terapia": 45.0,
+        "overall_accuracy": 39.380441384895725
+    }
+]

leaderboards/r_ldek_report_scores.json

@@ -0,0 +1,132 @@
+[
+    {
+        "model_name": "model-epfl-llm-meditron-7b",
+        "2008_jesień": 22.727272727272727,
+        "2012_wiosna": 23.711340206185564,
+        "2010_jesień": 19.597989949748744,
+        "2009_wiosna": 16.161616161616163,
+        "2010_wiosna": 16.0,
+        "2011_wiosna": 19.696969696969695,
+        "2012_jesień": 23.46938775510204,
+        "2011_jesień": 24.747474747474747,
+        "2009_jesień": 18.5,
+        "2022_wiosna": 22.727272727272727,
+        "2014_wiosna": 17.346938775510203,
+        "2022_jesien": 22.75132275132275,
+        "2014_jesien": 16.580310880829018,
+        "2024_wiosna": 22.916666666666664,
+        "2015_wiosna": 19.35483870967742,
+        "2023_wiosna": 21.025641025641026,
+        "2021_jesien": 24.873096446700508,
+        "2015_jesien": 18.947368421052634,
+        "2023_jesien": 22.164948453608247,
+        "2021_wiosna": 24.489795918367346,
+        "2013_wiosna": 18.04123711340206,
+        "2013_jesien": 17.894736842105264,
+        "overall_accuracy": 20.624563038918666
+    },
+    {
+        "model_name": "model-speakleash-Bielik-11B-v2.2-Instruct",
+        "2008_jesień": 45.45454545454545,
+        "2012_wiosna": 46.391752577319586,
+        "2010_jesień": 52.26130653266332,
+        "2009_wiosna": 51.515151515151516,
+        "2010_wiosna": 48.5,
+        "2011_wiosna": 41.91919191919192,
+        "2012_jesień": 57.6530612244898,
+        "2011_jesień": 45.95959595959596,
+        "2009_jesień": 49.5,
+        "2022_wiosna": 40.4040404040404,
+        "2014_wiosna": 34.69387755102041,
+        "2022_jesien": 43.91534391534391,
+        "2014_jesien": 44.04145077720207,
+        "2024_wiosna": 51.041666666666664,
+        "2015_wiosna": 43.54838709677419,
+        "2023_wiosna": 42.05128205128205,
+        "2021_jesien": 44.16243654822335,
+        "2015_jesien": 40.0,
+        "2023_jesien": 38.144329896907216,
+        "2021_wiosna": 50.0,
+        "2013_wiosna": 47.93814432989691,
+        "2013_jesien": 43.15789473684211,
+        "overall_accuracy": 45.583780004660916
+    },
+    {
+        "model_name": "model-meta-llama-Meta-Llama-3.1-8B-Instruct",
+        "2008_jesień": 41.41414141414141,
+        "2012_wiosna": 41.23711340206185,
+        "2010_jesień": 43.21608040201005,
+        "2009_wiosna": 47.97979797979798,
+        "2010_wiosna": 46.5,
+        "2011_wiosna": 33.33333333333333,
+        "2012_jesień": 48.46938775510204,
+        "2011_jesień": 41.41414141414141,
+        "2009_jesień": 54.50000000000001,
+        "2022_wiosna": 40.4040404040404,
+        "2014_wiosna": 33.6734693877551,
+        "2022_jesien": 41.7989417989418,
+        "2014_jesien": 42.487046632124354,
+        "2024_wiosna": 39.0625,
+        "2015_wiosna": 42.473118279569896,
+        "2023_wiosna": 45.64102564102564,
+        "2021_jesien": 40.609137055837564,
+        "2015_jesien": 41.578947368421055,
+        "2023_jesien": 42.2680412371134,
+        "2021_wiosna": 41.3265306122449,
+        "2013_wiosna": 41.23711340206185,
+        "2013_jesien": 45.78947368421053,
+        "overall_accuracy": 42.57748776508972
+    },
+    {
+        "model_name": "model-meta-llama-Meta-Llama-3.1-70B-Instruct",
+        "2008_jesień": 62.121212121212125,
+        "2012_wiosna": 61.34020618556701,
+        "2010_jesień": 62.8140703517588,
+        "2009_wiosna": 69.1919191919192,
+        "2010_wiosna": 73.0,
+        "2011_wiosna": 59.59595959595959,
+        "2012_jesień": 69.89795918367348,
+        "2011_jesień": 65.65656565656566,
+        "2009_jesień": 68.0,
+        "2022_wiosna": 68.68686868686868,
+        "2014_wiosna": 54.59183673469388,
+        "2022_jesien": 64.02116402116403,
+        "2014_jesien": 60.62176165803109,
+        "2024_wiosna": 61.979166666666664,
+        "2015_wiosna": 60.215053763440864,
+        "2023_wiosna": 63.07692307692307,
+        "2021_jesien": 63.95939086294417,
+        "2015_jesien": 58.42105263157895,
+        "2023_jesien": 61.34020618556701,
+        "2021_wiosna": 68.36734693877551,
+        "2013_wiosna": 62.371134020618555,
+        "2013_jesien": 57.36842105263158,
+        "overall_accuracy": 63.528315078070385
+    },
+    {
+        "model_name": "model-mistralai-Mistral-Large-Instruct-2407",
+        "2008_jesień": 61.61616161616161,
+        "2012_wiosna": 54.123711340206185,
+        "2010_jesień": 62.311557788944725,
+        "2009_wiosna": 67.67676767676768,
+        "2010_wiosna": 69.5,
+        "2011_wiosna": 56.56565656565656,
+        "2012_jesień": 66.3265306122449,
+        "2011_jesień": 58.080808080808076,
+        "2009_jesień": 63.0,
+        "2022_wiosna": 54.54545454545454,
+        "2014_wiosna": 54.08163265306123,
+        "2022_jesien": 57.14285714285714,
+        "2014_jesien": 56.476683937823836,
+        "2024_wiosna": 56.770833333333336,
+        "2015_wiosna": 58.602150537634415,
+        "2023_wiosna": 57.948717948717956,
+        "2021_jesien": 55.83756345177665,
+        "2015_jesien": 60.0,
+        "2023_jesien": 48.45360824742268,
+        "2021_wiosna": 57.14285714285714,
+        "2013_wiosna": 61.34020618556701,
+        "2013_jesien": 55.78947368421052,
+        "overall_accuracy": 58.820787695175945
+    }
+]

leaderboards/r_lek_report_scores.json

@@ -0,0 +1,132 @@
+[
+    {
+        "model_name": "model-epfl-llm-meditron-7b",
+        "2011_wiosna": 19.696969696969695,
+        "2010_wiosna": 26.5,
+        "2008_jesień": 23.85786802030457,
+        "2011_jesień": 23.232323232323232,
+        "2009_jesień": 22.33502538071066,
+        "2010_jesień": 19.696969696969695,
+        "2012_wiosna": 24.5,
+        "2009_wiosna": 24.120603015075375,
+        "2012_jesień": 19.897959183673468,
+        "2022_wiosna": 19.3717277486911,
+        "2014_wiosna": 20.0,
+        "2022_jesien": 28.654970760233915,
+        "2014_jesien": 23.958333333333336,
+        "2024_wiosna": 23.809523809523807,
+        "2015_wiosna": 18.781725888324875,
+        "2023_wiosna": 20.46783625730994,
+        "2021_jesien": 19.170984455958546,
+        "2015_jesien": 22.22222222222222,
+        "2023_jesien": 23.4375,
+        "2021_wiosna": 15.656565656565657,
+        "2013_wiosna": 16.145833333333336,
+        "2013_jesien": 21.465968586387437,
+        "overall_accuracy": 21.64338148235851
+    },
+    {
+        "model_name": "model-speakleash-Bielik-11B-v2.2-Instruct",
+        "2011_wiosna": 59.59595959595959,
+        "2010_wiosna": 62.0,
+        "2008_jesień": 59.89847715736041,
+        "2011_jesień": 60.1010101010101,
+        "2009_jesień": 58.37563451776649,
+        "2010_jesień": 63.13131313131313,
+        "2012_wiosna": 61.0,
+        "2009_wiosna": 63.31658291457286,
+        "2012_jesień": 63.26530612244898,
+        "2022_wiosna": 58.63874345549738,
+        "2014_wiosna": 57.948717948717956,
+        "2022_jesien": 69.00584795321637,
+        "2014_jesien": 63.541666666666664,
+        "2024_wiosna": 66.07142857142857,
+        "2015_wiosna": 59.390862944162436,
+        "2023_wiosna": 66.08187134502924,
+        "2021_jesien": 62.17616580310881,
+        "2015_jesien": 61.904761904761905,
+        "2023_jesien": 63.541666666666664,
+        "2021_wiosna": 65.15151515151516,
+        "2013_wiosna": 61.458333333333336,
+        "2013_jesien": 61.78010471204188,
+        "overall_accuracy": 62.064882784750175
+    },
+    {
+        "model_name": "model-meta-llama-Meta-Llama-3.1-8B-Instruct",
+        "2011_wiosna": 52.02020202020202,
+        "2010_wiosna": 53.5,
+        "2008_jesień": 50.25380710659898,
+        "2011_jesień": 57.07070707070707,
+        "2009_jesień": 45.68527918781726,
+        "2010_jesień": 43.93939393939394,
+        "2012_wiosna": 54.0,
+        "2009_wiosna": 45.22613065326633,
+        "2012_jesień": 53.06122448979592,
+        "2022_wiosna": 45.54973821989529,
+        "2014_wiosna": 52.307692307692314,
+        "2022_jesien": 53.80116959064327,
+        "2014_jesien": 54.166666666666664,
+        "2024_wiosna": 57.14285714285714,
+        "2015_wiosna": 54.82233502538071,
+        "2023_wiosna": 53.216374269005854,
+        "2021_jesien": 51.813471502590666,
+        "2015_jesien": 44.44444444444444,
+        "2023_jesien": 50.0,
+        "2021_wiosna": 47.97979797979798,
+        "2013_wiosna": 54.6875,
+        "2013_jesien": 51.832460732984295,
+        "overall_accuracy": 51.148472649775044
+    },
+    {
+        "model_name": "model-meta-llama-Meta-Llama-3.1-70B-Instruct",
+        "2011_wiosna": 77.27272727272727,
+        "2010_wiosna": 82.0,
+        "2008_jesień": 79.69543147208121,
+        "2011_jesień": 83.33333333333334,
+        "2009_jesień": 81.21827411167513,
+        "2010_jesień": 80.8080808080808,
+        "2012_wiosna": 80.0,
+        "2009_wiosna": 72.8643216080402,
+        "2012_jesień": 81.12244897959184,
+        "2022_wiosna": 79.05759162303664,
+        "2014_wiosna": 76.41025641025641,
+        "2022_jesien": 82.45614035087719,
+        "2014_jesien": 87.5,
+        "2024_wiosna": 86.90476190476191,
+        "2015_wiosna": 77.66497461928934,
+        "2023_wiosna": 80.7017543859649,
+        "2021_jesien": 81.34715025906736,
+        "2015_jesien": 78.83597883597884,
+        "2023_jesien": 83.33333333333334,
+        "2021_wiosna": 84.84848484848484,
+        "2013_wiosna": 77.60416666666666,
+        "2013_jesien": 78.01047120418848,
+        "overall_accuracy": 80.53516457494672
+    },
+    {
+        "model_name": "model-mistralai-Mistral-Large-Instruct-2407",
+        "2011_wiosna": 76.26262626262627,
+        "2010_wiosna": 76.5,
+        "2008_jesień": 76.6497461928934,
+        "2011_jesień": 79.29292929292929,
+        "2009_jesień": 71.57360406091371,
+        "2010_jesień": 74.24242424242425,
+        "2012_wiosna": 77.5,
+        "2009_wiosna": 72.8643216080402,
+        "2012_jesień": 76.53061224489795,
+        "2022_wiosna": 73.29842931937172,
+        "2014_wiosna": 80.0,
+        "2022_jesien": 77.19298245614034,
+        "2014_jesien": 76.5625,
+        "2024_wiosna": 87.5,
+        "2015_wiosna": 75.1269035532995,
+        "2023_wiosna": 80.11695906432749,
+        "2021_jesien": 76.16580310880829,
+        "2015_jesien": 71.42857142857143,
+        "2023_jesien": 76.5625,
+        "2021_wiosna": 79.29292929292929,
+        "2013_wiosna": 71.875,
+        "2013_jesien": 75.91623036649214,
+        "overall_accuracy": 76.39119109637699
+    }
+]

requirements.txt

@@ -0,0 +1,16 @@
+APScheduler
+black
+datasets
+gradio
+gradio[oauth]
+gradio_leaderboard==0.0.9
+gradio_client
+huggingface-hub>=0.18.0
+matplotlib
+numpy
+pandas
+python-dateutil
+tqdm
+transformers
+tokenizers>=0.15.0
+sentencepiece

src/abouts.py

@@ -0,0 +1,24 @@
+HEADER_TITLE = """
+<div style="display: flex; flex-wrap: wrap; justify-content: space-around; align-items: center;">
+    <div>
+        <h1 align="center" id="space-title">Adam Mickiewicz University's Center for Artificial Intelligence</h1>
+        <h2 align="center" id="space-subtitle">LLMs medical benchmark</h2>
+    </div>
+</div>
+"""
+
+LEADERBOARD_DESC = """## Test
+        """
+
+PES_DESC = """## ...
+        """
+
+LDEK_DESC = """## ...
+        """
+
+LEK_DESC = """## ...
+        """
+
+CITATION_LABEL = "TEST_LABEL"
+
+CITATION_CONTENT = "CONTENT"

src/envs.py

@@ -0,0 +1,25 @@
+import os
+
+from huggingface_hub import HfApi
+
+# Info to change for your repository
+# ----------------------------------
+TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+
+OWNER = "MCiesiolka" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+# ----------------------------------
+
+REPO_ID = f"{OWNER}/test_leaderboard"
+QUEUE_REPO = f"{OWNER}/requests"
+RESULTS_REPO = f"{OWNER}/results"
+
+# If you setup a cache later, just change HF_HOME
+CACHE_PATH=os.getenv("HF_HOME", ".")
+
+# Local caches
+EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
+EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
+EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
+EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
+
+API = HfApi(token=TOKEN)

src/structures/all_structure.py

@@ -0,0 +1,63 @@
+import pandas as pd
+from pathlib import Path
+
+abs_path = Path(__file__).parent.parent.parent
+
+def replace_models_names(model_name):
+    replaces = {'meta-llama': 'meta_llama',
+        'epfl-llm':'epfl_llm',
+        '01-ai':'01_ai'}
+    new_name = model_name.replace('model-', '')
+    for k, v in replaces.items():
+        if new_name.startswith(k):
+            new_name = new_name.replace(k, v)
+    new_name = new_name.replace('-','/',1)
+    new_name = new_name.replace('_','-',1)
+    new_name = f"[{new_name}](https://huggingface.co/{new_name})"
+    return new_name
+
+def load_json_data(file_path):
+    ALL_ACCS = pd.read_json(file_path)
+
+    for column in ALL_ACCS.columns:
+        if ALL_ACCS[column].apply(type).eq(dict).any():
+            ALL_ACCS[column] = ALL_ACCS[column].apply(str)
+
+    for column in ALL_ACCS.select_dtypes(include='number').columns:
+        ALL_ACCS[column] = ALL_ACCS[column].round(2)
+
+    return ALL_ACCS
+
+file_paths = [
+    str(abs_path / "leaderboards/pes_accuracy.json"),
+    str(abs_path / "leaderboards/r_ldek_report_scores.json"),
+    str(abs_path / "leaderboards/r_lek_report_scores.json"),
+]
+
+model_data = {}
+
+for file_path in file_paths:
+    ALL_ACCS = load_json_data(file_path)
+
+    for _, row in ALL_ACCS.iterrows():
+        model_name = replace_models_names(row["model_name"])
+        overall_accuracy = row["overall_accuracy"]
+
+        if model_name not in model_data:
+            model_data[model_name] = {"model_name": model_name}
+
+        file_key = file_path.split("/")[-1].replace(".json", "")  # Use file name as key
+        model_data[model_name][f"overall_acc_from_{file_key}"] = overall_accuracy
+
+ALL_ACCS = pd.DataFrame(list(model_data.values()))
+ALL_ACCS=ALL_ACCS.rename(columns={'overall_acc_from_pes_accuracy':'PES',
+    'overall_acc_from_r_ldek_report_scores':'LDEK',
+    'overall_acc_from_r_lek_report_scores':'LEK'})
+
+ALL_ACCS['Average'] = ALL_ACCS[['PES', 'LDEK', 'LEK']].mean(axis=1).round(2)
+columns = list(ALL_ACCS.columns)
+columns.insert(1, columns.pop(columns.index('Average')))
+ALL_ACCS = ALL_ACCS[columns]
+STYLED = ALL_ACCS.style.highlight_max(
+    color = '#ff7070',
+    subset=ALL_ACCS.columns[-4:]).format(precision=2)

src/structures/ldek_structure.py

@@ -0,0 +1,66 @@
+import pandas as pd
+from pathlib import Path
+
+abs_path = Path(__file__).parent.parent.parent
+
+def replace_models_names(model_name):
+    replaces = {'meta-llama': 'meta_llama',
+        'epfl-llm':'epfl_llm',
+        '01-ai':'01_ai'}
+    new_name = model_name.replace('model-', '')
+    for k, v in replaces.items():
+        if new_name.startswith(k):
+            new_name = new_name.replace(k, v)
+    new_name = new_name.replace('-','/',1)
+    new_name = new_name.replace('_','-',1)
+    new_name = f"[{new_name}](https://huggingface.co/{new_name})"
+    return new_name
+
+def generate_ORDER_LIST_LDEK_and_data_types(json_data):
+    ORDER_LIST_LDEK = ["model_name", "overall_accuracy"]
+    data_types = ["markdown", "number"]
+
+    for key in json_data.keys():
+        if key not in ["model_name", "overall_accuracy"]:
+            ORDER_LIST_LDEK.append(key)
+            data_types.append("number")
+    ORDER_LIST_LDEK[2:] = sorted(ORDER_LIST_LDEK[2:])
+    return ORDER_LIST_LDEK, data_types
+
+def filter_columns_ldek(column_choices):
+    selected_columns = [col for col in ORDER_LIST_LDEK if col in column_choices]
+    return LDEK_ACCS[selected_columns]
+
+def load_json_data(file_path, ORDER_LIST_LDEK):
+    LDEK_ACCS = pd.read_json(file_path)
+    for column in LDEK_ACCS.columns:
+        if LDEK_ACCS[column].apply(type).eq(dict).any():
+            LDEK_ACCS[column] = LDEK_ACCS[column].apply(str)
+
+    LDEK_ACCS["model_name"] = LDEK_ACCS["model_name"].apply(
+        lambda name: replace_models_names(name)
+    )
+
+    for column in LDEK_ACCS.select_dtypes(include='number').columns:
+            LDEK_ACCS[column] = LDEK_ACCS[column].round(2)
+    ordered_columns = [col for col in ORDER_LIST_LDEK if col in LDEK_ACCS.columns]
+    LDEK_ACCS = LDEK_ACCS[ordered_columns]
+
+    if "Score" in LDEK_ACCS.columns:
+        LDEK_ACCS = LDEK_ACCS.sort_values(by="Score", ascending=False)
+
+    return LDEK_ACCS
+
+
+file_path = str(abs_path / "leaderboards/r_ldek_report_scores.json")
+with open(file_path, 'r', encoding='utf-8') as file:
+    sample_data = pd.read_json(file_path).iloc[0].to_dict()  # Load the first row as a dict
+
+ORDER_LIST_LDEK, DATA_TYPES_LDEK = generate_ORDER_LIST_LDEK_and_data_types(sample_data)
+LDEK_ACCS = load_json_data(file_path, ORDER_LIST_LDEK)
+# LDEK_ACCS = LDEK_ACCS.style.highlight_max(
+#     color = '#ff7070',
+#     subset=LDEK_ACCS.columns[1:]).format(precision=2)
+COLUMN_HEADERS_LDEK = ORDER_LIST_LDEK
+
+print(ORDER_LIST_LDEK)

src/structures/lek_structure.py

@@ -0,0 +1,64 @@
+import pandas as pd
+from pathlib import Path
+
+abs_path = Path(__file__).parent.parent.parent
+
+def replace_models_names(model_name):
+    replaces = {'meta-llama': 'meta_llama',
+        'epfl-llm':'epfl_llm',
+        '01-ai':'01_ai'}
+    new_name = model_name.replace('model-', '')
+    for k, v in replaces.items():
+        if new_name.startswith(k):
+            new_name = new_name.replace(k, v)
+    new_name = new_name.replace('-','/',1)
+    new_name = new_name.replace('_','-',1)
+    new_name = f"[{new_name}](https://huggingface.co/{new_name})"
+    return new_name
+
+def generate_ORDER_LIST_LEK_and_data_types(json_data):
+    ORDER_LIST_LEK = ["model_name", "overall_accuracy"]
+    data_types = ["markdown", "number"]
+
+    for key in json_data.keys():
+        if key not in ["model_name", "overall_accuracy"]:
+            ORDER_LIST_LEK.append(key)
+            data_types.append("number")
+    ORDER_LIST_LEK[2:] = sorted(ORDER_LIST_LEK[2:])
+    return ORDER_LIST_LEK, data_types
+
+def filter_columns_lek(column_choices):
+    selected_columns = [col for col in ORDER_LIST_LEK if col in column_choices]
+    return LEK_ACCS[selected_columns]
+
+def load_json_data(file_path, ORDER_LIST_LEK):
+    LEK_ACCS = pd.read_json(file_path)
+    for column in LEK_ACCS.columns:
+        if LEK_ACCS[column].apply(type).eq(dict).any():
+            LEK_ACCS[column] = LEK_ACCS[column].apply(str)
+
+    LEK_ACCS["model_name"] = LEK_ACCS["model_name"].apply(
+        lambda name: replace_models_names(name)
+    )
+
+    for column in LEK_ACCS.select_dtypes(include='number').columns:
+            LEK_ACCS[column] = LEK_ACCS[column].round(2)
+    ordered_columns = [col for col in ORDER_LIST_LEK if col in LEK_ACCS.columns]
+    LEK_ACCS = LEK_ACCS[ordered_columns]
+
+    if "Score" in LEK_ACCS.columns:
+        LEK_ACCS = LEK_ACCS.sort_values(by="Score", ascending=False)
+
+    return LEK_ACCS
+
+
+file_path = str(abs_path / "leaderboards/r_lek_report_scores.json")
+with open(file_path, 'r', encoding='utf-8') as file:
+    sample_data = pd.read_json(file_path).iloc[0].to_dict()  # Load the first row as a dict
+
+ORDER_LIST_LEK, DATA_TYPES_LEK = generate_ORDER_LIST_LEK_and_data_types(sample_data)
+LEK_ACCS = load_json_data(file_path, ORDER_LIST_LEK)
+# LEK_ACCS = LEK_ACCS.reindex(sorted(LEK_ACCS.columns), axis=1)
+COLUMN_HEADERS_LEK = ORDER_LIST_LEK
+
+print(ORDER_LIST_LEK)

src/structures/pes_structure.py

@@ -0,0 +1,72 @@
+import pandas as pd
+from pathlib import Path
+
+abs_path = Path(__file__).parent.parent.parent
+
+def replace_models_names(model_name):
+    replaces = {'meta-llama': 'meta_llama',
+        'epfl-llm':'epfl_llm',
+        '01-ai':'01_ai'}
+    new_name = model_name.replace('model-', '')
+    for k, v in replaces.items():
+        if new_name.startswith(k):
+            new_name = new_name.replace(k, v)
+    new_name = new_name.replace('-','/',1)
+    new_name = new_name.replace('_','-',1)
+    new_name = f"[{new_name}](https://huggingface.co/{new_name})"
+    return new_name
+
+
+def generate_order_list_and_data_types(json_data):
+    order_list = ["model_name", "overall_accuracy"]
+    data_types = ["markdown", "number"]
+
+    for key in json_data.keys():
+        if key not in ["model_name", "overall_accuracy"]:
+            order_list.append(key)
+            data_types.append("number")
+    order_list[2:] = sorted(order_list[2:])
+    return order_list, data_types
+
+def filter_data(selected_columns, search_query):
+    df = PES_ACCS[selected_columns]
+    if search_query:
+        df = df[df['model_name'].str.contains(search_query, case=False, na=False)]
+    return df
+
+def filter_columns(column_choices):
+    selected_columns = [col for col in ORDER_LIST if col in column_choices]
+    return PES_ACCS[selected_columns]
+
+def load_json_data(file_path, order_list):
+    PES_ACCS = pd.read_json(file_path)
+    for column in PES_ACCS.columns:
+        if PES_ACCS[column].apply(type).eq(dict).any():
+            PES_ACCS[column] = PES_ACCS[column].apply(str)
+
+    PES_ACCS["model_name"] = PES_ACCS["model_name"].apply(
+        lambda name: replace_models_names(name)
+    )
+
+    for column in PES_ACCS.select_dtypes(include='number').columns:
+            PES_ACCS[column] = PES_ACCS[column].round(2)
+    ordered_columns = [col for col in order_list if col in PES_ACCS.columns]
+    PES_ACCS = PES_ACCS[ordered_columns]
+
+    if "Score" in PES_ACCS.columns:
+        PES_ACCS = PES_ACCS.sort_values(by="Score", ascending=False)
+
+    return PES_ACCS
+file_path = str(abs_path / "leaderboards/pes_accuracy.json")
+
+with open(file_path, 'r', encoding='utf-8') as file:
+    sample_data = pd.read_json(file_path).iloc[0].to_dict()  # Load the first row as a dict
+
+ORDER_LIST, DATA_TYPES = generate_order_list_and_data_types(sample_data)
+PES_ACCS = load_json_data(file_path, ORDER_LIST)
+# PES_ACCS = PES_ACCS.style.highlight_max(
+#     color = '#ff7070',
+#     subset=PES_ACCS.columns[1:]).format(precision=2)
+COLUMN_HEADERS = ORDER_LIST
+
+print('test')

src/styles.py

@@ -0,0 +1,108 @@
+custom_css = """
+.markdown-text {
+    font-size: 16px !important;
+}
+#models-to-add-text {
+    font-size: 18px !important;
+}
+#citation-button span {
+    font-size: 16px !important;
+}
+#citation-button textarea {
+    font-size: 16px !important;
+}
+#citation-button > label > button {
+    margin: 6px;
+    transform: scale(1.3);
+}
+#leaderboard-table {
+    margin-top: 15px
+}
+#leaderboard-table-lite {
+    margin-top: 15px
+}
+#search-bar-table-box > div:first-child {
+    background: none;
+    border: none;
+}
+
+#search-bar {
+    padding: 0px;
+}
+/* Hides the final AutoEvalColumn */
+#llm-benchmark-tab-table table td:last-child,
+#llm-benchmark-tab-table table th:last-child {
+    display: none;
+}
+/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
+table td:first-child,
+table th:first-child {
+    max-width: 400px;
+    overflow: auto;
+    white-space: nowrap;
+}
+table > tbody > tr > td:nth-child(3) > div {
+    overflow-x: auto;
+    width: 450px;
+}
+tbody span {
+    -webkit-user-select: text;
+    -moz-user-select: text;
+    -ms-user-select: text;
+    user-select: text;
+}
+.tab-buttons button {
+    font-size: 20px;
+}
+#scale-logo {
+    border-style: none !important;
+    box-shadow: none;
+    display: block;
+    margin-left: auto;
+    margin-right: auto;
+    max-width: 600px;
+}
+#scale-logo .download {
+    display: none;
+}
+#filter_type{
+    border: 0;
+    padding-left: 0;
+    padding-top: 0;
+}
+#filter_type label {
+    display: flex;
+}
+#filter_type label > span{
+    margin-top: var(--spacing-lg);
+    margin-right: 0.5em;
+}
+#filter_type label > .wrap{
+    width: 103px;
+}
+#filter_type label > .wrap .wrap-inner{
+    padding: 2px;
+}
+#filter_type label > .wrap .wrap-inner input{
+    width: 1px
+}
+#filter-columns-type{
+    border:0;
+    padding:0.5;
+}
+#filter-columns-size{
+    border:0;
+    padding:0.5;
+}
+#box-filter > .form{
+    border: 0
+}
+"""
+
+get_window_url_params = """
+    function(url_params) {
+        const params = new URLSearchParams(window.location.search);
+        url_params = Object.fromEntries(params);
+        return url_params;
+    }
+    """

src/utils.py

@@ -0,0 +1 @@
+