| { | |
| "model": { | |
| "model": "CohereForAI/aya-expanse-8b", | |
| "api": "hf", | |
| "architecture": "CohereForCausalLM", | |
| "max_length": 8192, | |
| "dtype": "float16", | |
| "type": "instruction-tuned", | |
| "num_parameters": "8b" | |
| }, | |
| "results": [ | |
| { | |
| "name": "belebele_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.7355555555555555, | |
| "acc_norm": 0.7355555555555555 | |
| }, | |
| { | |
| "name": "exams_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.3155216284987277, | |
| "acc_norm": 0.3460559796437659 | |
| }, | |
| { | |
| "name": "check_worthiness", | |
| "task": "multiple_choice", | |
| "acc": 0.4026508226691042, | |
| "acc_norm": 0.6224862888482633 | |
| }, | |
| { | |
| "name": "gecturk_generation", | |
| "task": "grammatical_error_correction", | |
| "exact_match": 0.0018296499590736194 | |
| }, | |
| { | |
| "name": "ironytr", | |
| "task": "text_classification", | |
| "acc": 0.505, | |
| "acc_norm": 0.49833333333333335 | |
| }, | |
| { | |
| "name": "mkqa_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.06954720331459012, | |
| "f1": 0.13476533908972033 | |
| }, | |
| { | |
| "name": "mlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.363610486561065, | |
| "rouge2": 0.21362825588593481, | |
| "rougeL": 0.29773476508614094 | |
| }, | |
| { | |
| "name": "mnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.3078, | |
| "acc_norm": 0.35 | |
| }, | |
| { | |
| "name": "news_cat", | |
| "task": "text_classification", | |
| "acc": 0.76, | |
| "acc_norm": 0.58 | |
| }, | |
| { | |
| "name": "offenseval_tr", | |
| "task": "text_classification", | |
| "acc": 0.2675736961451247, | |
| "acc_norm": 0.7956349206349206 | |
| }, | |
| { | |
| "name": "relevance_judgment", | |
| "task": "multiple_choice", | |
| "acc": 0.5877513711151737, | |
| "acc_norm": 0.579981718464351 | |
| }, | |
| { | |
| "name": "snli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.344, | |
| "acc_norm": 0.3435 | |
| }, | |
| { | |
| "name": "sts_tr", | |
| "task": "text_classification", | |
| "acc": 0.2095721537345903, | |
| "acc_norm": 0.21029731689630166 | |
| }, | |
| { | |
| "name": "tquad", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.13452914798206278, | |
| "f1": 0.435087842533856 | |
| }, | |
| { | |
| "name": "turkish_plu_goal_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.4062126642771804, | |
| "acc_norm": 0.3930704898446834 | |
| }, | |
| { | |
| "name": "turkish_plu_next_event_prediction", | |
| "task": "multiple_choice", | |
| "acc": 0.4900763358778626, | |
| "acc_norm": 0.5465648854961832 | |
| }, | |
| { | |
| "name": "turkish_plu_step_inference", | |
| "task": "multiple_choice", | |
| "acc": 0.3464052287581699, | |
| "acc_norm": 0.4395424836601307 | |
| }, | |
| { | |
| "name": "turkish_plu_step_ordering", | |
| "task": "multiple_choice", | |
| "acc": 0.5935357492654261, | |
| "acc_norm": 0.5935357492654261 | |
| }, | |
| { | |
| "name": "wiki_lingua_tr", | |
| "task": "summarization", | |
| "rouge1": 0.3064320242538614, | |
| "rouge2": 0.1340385267540697, | |
| "rougeL": 0.24764232131755232 | |
| }, | |
| { | |
| "name": "wmt-tr-en-prompt", | |
| "task": "machine_translation", | |
| "wer": 0.7822550373875778, | |
| "bleu": 0.17034711245148307 | |
| }, | |
| { | |
| "name": "xcopa_tr", | |
| "task": "multiple_choice", | |
| "acc": 0.578, | |
| "acc_norm": 0.578 | |
| }, | |
| { | |
| "name": "xlsum_tr", | |
| "task": "summarization", | |
| "rouge1": 0.26621653203927675, | |
| "rouge2": 0.133428873146516, | |
| "rougeL": 0.2083669711429916 | |
| }, | |
| { | |
| "name": "xnli_tr", | |
| "task": "natural_language_inference", | |
| "acc": 0.4919678714859438, | |
| "acc_norm": 0.4919678714859438 | |
| }, | |
| { | |
| "name": "xquad_tr", | |
| "task": "extractive_question_answering", | |
| "exact_match": 0.2495798319327731, | |
| "f1": 0.4735125568867167 | |
| } | |
| ] | |
| } |