|
from dataclasses import dataclass |
|
from enum import Enum |
|
|
|
@dataclass(frozen=True) |
|
class Task: |
|
benchmark: str |
|
metric: str |
|
col_name: str |
|
type: str |
|
baseline: float = 0.0 |
|
|
|
|
|
|
|
|
|
class Tasks(Enum): |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
task30a = Task("polish_pes", "exact_match,score-first", "pes", "generate_until", 0.0) |
|
task60 = Task("polish_pes_medycyna_rodzinna", "exact_match,score-first", "medycyna_rodzinna", "generate_until", 0.0) |
|
task73 = Task("polish_pes_pediatria", "exact_match,score-first", "pediatria", "generate_until", 0.0) |
|
|
|
task30 = Task("polish_pes_alergologia", "exact_match,score-first", "alergologia", "generate_until", 0.0) |
|
task31 = Task("polish_pes_anestezjologia", "exact_match,score-first", "anestezjologia", "generate_until", 0.0) |
|
task32 = Task("polish_pes_angiologia", "exact_match,score-first", "angiologia", "generate_until", 0.0) |
|
task33 = Task("polish_pes_balneologia_i_medycyna_fizykalna", "exact_match,score-first", "balneologia_i_medycyna_fizykalna", "generate_until", 0.0) |
|
task34 = Task("polish_pes_chirurgia_dziecieca", "exact_match,score-first", "chirurgia_dziecieca", "generate_until", 0.0) |
|
task35 = Task("polish_pes_chirurgia_naczyniowa", "exact_match,score-first", "chirurgia_naczyniowa", "generate_until", 0.0) |
|
task36 = Task("polish_pes_chirurgia_ogolna", "exact_match,score-first", "chirurgia_ogolna", "generate_until", 0.0) |
|
task37 = Task("polish_pes_chirurgia_onkologiczna", "exact_match,score-first", "chirurgia_onkologiczna", "generate_until", 0.0) |
|
task38 = Task("polish_pes_chirurgia_stomatologiczna", "exact_match,score-first", "chirurgia_stomatologiczna", "generate_until", 0.0) |
|
task39 = Task("polish_pes_chirurgia_szczekowo-twarzowa", "exact_match,score-first", "chirurgia_szczekowo-twarzowa", "generate_until", 0.0) |
|
task40 = Task("polish_pes_choroby_pluc", "exact_match,score-first", "choroby_pluc", "generate_until", 0.0) |
|
task41 = Task("polish_pes_choroby_pluc_dzieci", "exact_match,score-first", "choroby_pluc_dzieci", "generate_until", 0.0) |
|
task42 = Task("polish_pes_choroby_wewnetrzne", "exact_match,score-first", "choroby_wewnetrzne", "generate_until", 0.0) |
|
task43 = Task("polish_pes_choroby_zakazne", "exact_match,score-first", "choroby_zakazne", "generate_until", 0.0) |
|
task44 = Task("polish_pes_dermatologia_i_wenerologia", "exact_match,score-first", "dermatologia_i_wenerologia", "generate_until", 0.0) |
|
task45 = Task("polish_pes_diabetologia", "exact_match,score-first", "diabetologia", "generate_until", 0.0) |
|
task46 = Task("polish_pes_endokrynologia", "exact_match,score-first", "endokrynologia", "generate_until", 0.0) |
|
task47 = Task("polish_pes_endokrynologia_ginekologiczna_i_rozrodczosc", "exact_match,score-first", "endokrynologia_ginekologiczna_i_rozrodczosc", "generate_until", 0.0) |
|
task48 = Task("polish_pes_endokrynologia_i_diabetologia_dziecieca", "exact_match,score-first", "endokrynologia_i_diabetologia_dziecieca", "generate_until", 0.0) |
|
task49 = Task("polish_pes_gastroenterologia", "exact_match,score-first", "gastroenterologia", "generate_until", 0.0) |
|
task50 = Task("polish_pes_gastroenterologia_dziecieca", "exact_match,score-first", "gastroenterologia_dziecieca", "generate_until", 0.0) |
|
task51 = Task("polish_pes_geriatria", "exact_match,score-first", "geriatria", "generate_until", 0.0) |
|
task52 = Task("polish_pes_ginekologia_onkologiczna", "exact_match,score-first", "ginekologia_onkologiczna", "generate_until", 0.0) |
|
task53 = Task("polish_pes_hematologia", "exact_match,score-first", "hematologia", "generate_until", 0.0) |
|
task54 = Task("polish_pes_hipertensjologia", "exact_match,score-first", "hipertensjologia", "generate_until", 0.0) |
|
task55 = Task("polish_pes_kardiochirurgia", "exact_match,score-first", "kardiochirurgia", "generate_until", 0.0) |
|
task56 = Task("polish_pes_kardiologia", "exact_match,score-first", "kardiologia", "generate_until", 0.0) |
|
task57 = Task("polish_pes_medycyna_pracy", "exact_match,score-first", "medycyna_pracy", "generate_until", 0.0) |
|
task58 = Task("polish_pes_medycyna_paliatywna", "exact_match,score-first", "medycyna_paliatywna", "generate_until", 0.0) |
|
task59 = Task("polish_pes_medycyna_ratunkowa", "exact_match,score-first", "medycyna_ratunkowa", "generate_until", 0.0) |
|
|
|
task61 = Task("polish_pes_medycyna_sportowa", "exact_match,score-first", "medycyna_sportowa", "generate_until", 0.0) |
|
task62 = Task("polish_pes_nefrologia", "exact_match,score-first", "nefrologia", "generate_until", 0.0) |
|
task63 = Task("polish_pes_neonatologia", "exact_match,score-first", "neonatologia", "generate_until", 0.0) |
|
task64 = Task("polish_pes_neurochirurgia", "exact_match,score-first", "neurochirurgia", "generate_until", 0.0) |
|
task65 = Task("polish_pes_neurologia", "exact_match,score-first", "neurologia", "generate_until", 0.0) |
|
task66 = Task("polish_pes_neurologia_dziecieca", "exact_match,score-first", "neurologia_dziecieca", "generate_until", 0.0) |
|
task67 = Task("polish_pes_okulistyka", "exact_match,score-first", "okulistyka", "generate_until", 0.0) |
|
task68 = Task("polish_pes_onkologia_kliniczna", "exact_match,score-first", "onkologia_kliniczna", "generate_until", 0.0) |
|
task69 = Task("polish_pes_ortodoncja", "exact_match,score-first", "ortodoncja", "generate_until", 0.0) |
|
task70 = Task("polish_pes_ortopedia", "exact_match,score-first", "ortopedia", "generate_until", 0.0) |
|
task71 = Task("polish_pes_otolaryngologia", "exact_match,score-first", "otolaryngologia", "generate_until", 0.0) |
|
task72 = Task("polish_pes_patomorfologia", "exact_match,score-first", "patomorfologia", "generate_until", 0.0) |
|
|
|
task74 = Task("polish_pes_perinatologia", "exact_match,score-first", "perinatologia", "generate_until", 0.0) |
|
task75 = Task("polish_pes_periodontologia", "exact_match,score-first", "periodontologia", "generate_until", 0.0) |
|
task76 = Task("polish_pes_poloznictwo_i_ginekologia", "exact_match,score-first", "poloznictwo_i_ginekologia", "generate_until", 0.0) |
|
task77 = Task("polish_pes_protetyka_stomatologiczna", "exact_match,score-first", "protetyka_stomatologiczna", "generate_until", 0.0) |
|
task78 = Task("polish_pes_psychiatria", "exact_match,score-first", "psychiatria", "generate_until", 0.0) |
|
task79 = Task("polish_pes_psychiatria_dzieci_i_mlodziezy", "exact_match,score-first", "psychiatria_dzieci_i_mlodziezy", "generate_until", 0.0) |
|
task80 = Task("polish_pes_radiologia_i_diagnostyka_obrazowa", "exact_match,score-first", "radiologia_i_diagnostyka_obrazowa", "generate_until", 0.0) |
|
task81 = Task("polish_pes_radioterapia_onkologiczna", "exact_match,score-first", "radioterapia_onkologiczna", "generate_until", 0.0) |
|
task82 = Task("polish_pes_rehabilitacja_medyczna", "exact_match,score-first", "rehabilitacja_medyczna", "generate_until", 0.0) |
|
task83 = Task("polish_pes_reumatologia", "exact_match,score-first", "reumatologia", "generate_until", 0.0) |
|
task84 = Task("polish_pes_stomatologia_dziecieca", "exact_match,score-first", "stomatologia_dziecieca", "generate_until", 0.0) |
|
task85 = Task("polish_pes_stomatologia_zachowawcza", "exact_match,score-first", "stomatologia_zachowawcza", "generate_until", 0.0) |
|
task86 = Task("polish_pes_transplantologia_kliniczna", "exact_match,score-first", "transplantologia_kliniczna", "generate_until", 0.0) |
|
|
|
|
|
g_tasks = [task.value.benchmark for task in Tasks if task.value.type == "generate_until"] |
|
mc_tasks = [task.value.benchmark for task in Tasks if task.value.type == "multiple_choice"] |
|
rag_tasks = ['polish_polqa_reranking_multiple_choice', 'polish_polqa_open_book', 'polish_poquad_open_book'] |
|
all_tasks = g_tasks + mc_tasks |
|
|
|
NUM_FEWSHOT = 0 |
|
|
|
|
|
|
|
|
|
|
|
TITLE = """ |
|
<div style="display: flex; flex-wrap: wrap; justify-content: space-around;"> |
|
<img src="https://speakleash.org/wp-content/uploads/2023/09/SpeakLeash_logo.svg"> |
|
<div> |
|
<h1 align="center" id="space-title">Polish Medical Leaderboard</h1> |
|
<h2 align="center" id="space-subtitle">Leaderboard was created as part of an open-science project SpeakLeash.org</h2> |
|
</div> |
|
</div> |
|
""" |
|
|
|
|
|
INTRODUCTION_TEXT = f""" |
|
The leaderboard evaluates language models on a set of Polish tasks. The tasks are designed to test the models' ability to understand and generate Polish text. The leaderboard is designed to be a benchmark for the Polish language model community, and to help researchers and practitioners understand the capabilities of different models. |
|
For now, models are tested without theirs templates. |
|
|
|
Almost every task has two versions: regex and multiple choice. |
|
* _g suffix means that a model needs to generate an answer (only suitable for instructions-based models) |
|
* _mc suffix means that a model is scored against every possible class (suitable also for base models) |
|
|
|
Average columns are normalized against scores by "Baseline (majority class)". |
|
|
|
* `,chat` suffix means that a model is tested using chat templates |
|
* `,chat,multiturn` suffix means that a model is tested using chat templates and fewshot examples are treated as a multi-turn conversation |
|
* 🚧 prefix means that not all tasks were calculated and the average scores are not accurate |
|
|
|
We gratefully acknowledge Polish high-performance computing infrastructure PLGrid (HPC Centers: ACK Cyfronet AGH) for providing computer facilities and support within computational grant no. PLG/2024/016951. |
|
""" |
|
|
|
|
|
LLM_BENCHMARKS_TEXT = f""" |
|
## Do you want to add your model to the leaderboard? |
|
|
|
Contact with me: [LinkedIn](https://www.linkedin.com/in/wrobelkrzysztof/) |
|
|
|
or join our [Discord SpeakLeash](https://discord.gg/FfYp4V6y3R) |
|
|
|
## TODO |
|
|
|
* fix long model names |
|
* add inference time |
|
* add more tasks |
|
* fix scrolling on Firefox |
|
|
|
## Tasks |
|
|
|
Tasks taken into account while calculating averages: |
|
* Average: {', '.join(all_tasks)} |
|
* Avg g: {', '.join(g_tasks)} |
|
* Avg mc: {', '.join(mc_tasks)} |
|
* Avg RAG: {', '.join(rag_tasks)} |
|
|
|
| Task | Dataset | Metric | Type | |
|
|---------------------------------|---------------------------------------|-----------|-----------------| |
|
| polemo2_in | allegro/klej-polemo2-in | accuracy | generate_until | |
|
| polemo2_in_mc | allegro/klej-polemo2-in | accuracy | multiple_choice | |
|
| polemo2_out | allegro/klej-polemo2-out | accuracy | generate_until | |
|
| polemo2_out_mc | allegro/klej-polemo2-out | accuracy | multiple_choice | |
|
| 8tags_mc | sdadas/8tags | accuracy | multiple_choice | |
|
| 8tags_g | sdadas/8tags | accuracy | generate_until | |
|
| belebele_mc | facebook/belebele | accuracy | multiple_choice | |
|
| belebele_g | facebook/belebele | accuracy | generate_until | |
|
| dyk_mc | allegro/klej-dyk | binary F1 | multiple_choice | |
|
| dyk_g | allegro/klej-dyk | binary F1 | generate_until | |
|
| ppc_mc | sdadas/ppc | accuracy | multiple_choice | |
|
| ppc_g | sdadas/ppc | accuracy | generate_until | |
|
| psc_mc | allegro/klej-psc | binary F1 | multiple_choice | |
|
| psc_g | allegro/klej-psc | binary F1 | generate_until | |
|
| cbd_mc | ptaszynski/PolishCyberbullyingDataset | macro F1 | multiple_choice | |
|
| cbd_g | ptaszynski/PolishCyberbullyingDataset | macro F1 | generate_until | |
|
| klej_ner_mc | allegro/klej-nkjp-ner | accuracy | multiple_choice | |
|
| klej_ner_g | allegro/klej-nkjp-ner | accuracy | generate_until | |
|
| polqa_reranking_mc | ipipan/polqa | accuracy | multiple_choice | |
|
| polqa_open_book_g | ipipan/polqa | levenshtein | generate_until | |
|
| polqa_closed_book_g | ipipan/polqa | levenshtein | generate_until | |
|
| poleval2018_task3_test_10k | enelpol/poleval2018_task3_test_10k | word perplexity | other | |
|
| polish_poquad_open_book | enelpol/poleval2018_task3_test_10k | levenshtein | generate_until | |
|
| polish_eq_bench_first_turn | speakleash/EQ-Bench-PL | eq_bench | generate_until | |
|
| polish_eq_bench | speakleash/EQ-Bench-PL | eq_bench | generate_until | |
|
|
|
## Reproducibility |
|
To reproduce our results, you need to clone the repository: |
|
|
|
``` |
|
git clone https://github.com/speakleash/lm-evaluation-harness.git -b polish3 |
|
cd lm-evaluation-harness |
|
pip install -e . |
|
``` |
|
|
|
and run benchmark for 0-shot and 5-shot: |
|
|
|
``` |
|
lm_eval --model hf --model_args pretrained=speakleash/Bielik-7B-Instruct-v0.1 --tasks polish_generate --num_fewshot 0 --output_path results/ --log_samples |
|
lm_eval --model hf --model_args pretrained=speakleash/Bielik-7B-Instruct-v0.1 --tasks polish_mc --num_fewshot 0 --output_path results/ --log_samples |
|
lm_eval --model hf --model_args pretrained=speakleash/Bielik-7B-Instruct-v0.1 --tasks polish_generate_few --num_fewshot 5 --output_path results/ --log_samples |
|
lm_eval --model hf --model_args pretrained=speakleash/Bielik-7B-Instruct-v0.1 --tasks polish_mc --num_fewshot 5 --output_path results/ --log_samples |
|
``` |
|
|
|
With chat templates: |
|
``` |
|
lm_eval --model hf --model_args pretrained=speakleash/Bielik-7B-Instruct-v0.1 --tasks polish_generate --num_fewshot 0 --output_path results/ --log_samples --apply_chat_template |
|
lm_eval --model hf --model_args pretrained=speakleash/Bielik-7B-Instruct-v0.1 --tasks polish_mc --num_fewshot 0 --output_path results/ --log_samples --apply_chat_template |
|
lm_eval --model hf --model_args pretrained=speakleash/Bielik-7B-Instruct-v0.1 --tasks polish_generate_few --num_fewshot 5 --output_path results/ --log_samples --apply_chat_template |
|
lm_eval --model hf --model_args pretrained=speakleash/Bielik-7B-Instruct-v0.1 --tasks polish_mc --num_fewshot 5 --output_path results/ --log_samples --apply_chat_template |
|
``` |
|
|
|
## List of Polish models |
|
|
|
* speakleash/Bielik-7B-Instruct-v0.1 |
|
* speakleash/Bielik-7B-v0.1 |
|
* Azurro/APT3-1B-Base |
|
* Azurro/APT3-1B-Instruct-v1 |
|
* Voicelab/trurl-2-7b |
|
* Voicelab/trurl-2-13b-academic |
|
* OPI-PG/Qra-1b |
|
* OPI-PG/Qra-7b |
|
* OPI-PG/Qra-13b |
|
* szymonrucinski/Curie-7B-v1 |
|
* sdadas/polish-gpt2-xl |
|
|
|
### List of multilingual models |
|
|
|
* meta-llama/Llama-2-7b-chat-hf |
|
* mistralai/Mistral-7B-Instruct-v0.1 |
|
* HuggingFaceH4/zephyr-7b-beta |
|
* HuggingFaceH4/zephyr-7b-alpha |
|
* internlm/internlm2-chat-7b-sft |
|
* internlm/internlm2-chat-7b |
|
* mistralai/Mistral-7B-Instruct-v0.2 |
|
* teknium/OpenHermes-2.5-Mistral-7B |
|
* openchat/openchat-3.5-1210 |
|
* Nexusflow/Starling-LM-7B-beta |
|
* openchat/openchat-3.5-0106 |
|
* berkeley-nest/Starling-LM-7B-alpha |
|
* upstage/SOLAR-10.7B-Instruct-v1.0 |
|
* meta-llama/Llama-2-7b-hf |
|
* internlm/internlm2-base-7b |
|
* mistralai/Mistral-7B-v0.1 |
|
* internlm/internlm2-7b |
|
* alpindale/Mistral-7B-v0.2-hf |
|
* internlm/internlm2-1_8b |
|
|
|
|
|
|
|
""" |
|
|
|
EVALUATION_QUEUE_TEXT = """ |
|
## Some good practices before submitting a model |
|
|
|
### 1) Make sure you can load your model and tokenizer using AutoClasses: |
|
```python |
|
from transformers import AutoConfig, AutoModel, AutoTokenizer |
|
config = AutoConfig.from_pretrained("your model name", revision=revision) |
|
model = AutoModel.from_pretrained("your model name", revision=revision) |
|
tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision) |
|
``` |
|
If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded. |
|
|
|
Note: make sure your model is public! |
|
Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted! |
|
|
|
### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index) |
|
It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`! |
|
|
|
### 3) Make sure your model has an open license! |
|
This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗 |
|
|
|
### 4) Fill up your model card |
|
When we add extra information about models to the leaderboard, it will be automatically taken from the model card |
|
|
|
## In case of model failure |
|
If your model is displayed in the `FAILED` category, its execution stopped. |
|
Make sure you have followed the above steps first. |
|
If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task). |
|
""" |
|
|
|
CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results" |
|
CITATION_BUTTON_TEXT = r""" |
|
@misc{open-pl-llm-leaderboard, |
|
title = {Open PL LLM Leaderboard}, |
|
author = {Wróbel, Krzysztof and {SpeakLeash Team} and {Cyfronet Team}}, |
|
year = 2024, |
|
publisher = {Hugging Face}, |
|
howpublished = "\url{https://huggingface.co/spaces/speakleash/open_pl_llm_leaderboard}" |
|
} |
|
""" |
|
|
