Model: OpenLLM-Ro/RoLlama2-7b-Base-2024-05-14
Author: OpenLLM-Ro
Pipeline: text-generation
Library: transformers
Tags: transformers, pytorch, llama, text-generation, ro, dataset:uonlp/CulturaX, arxiv:2406.18266, base_model:meta-llama/Llama-2-7b-hf, base_model:finetune:meta-llama/Llama-2-7b-hf, license:llama2, model-index, text-generation-inference, endpoints_compatible, region:us
Base models (finetune): id: meta-llama/Llama-2-7b-hf
Card metadata: license: llama2
language: ro
base_model: meta-llama/Llama-2-7b-hf
datasets: uonlp/CulturaX
model-index: name: OpenLLM-Ro/RoLlama2-7b-Base-2024-05-14
results: task: type: text-generation
dataset: name: Romanian_Academic_Benchmarks
type: Romanian_Academic_Benchmarks
metrics: name: Average accuracy
type: accuracy
value: 38.03
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_arc_challenge
type: OpenLLM-Ro/ro_arc_challenge
metrics: name: Average accuracy
type: accuracy
value: 37.95
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_mmlu
type: OpenLLM-Ro/ro_mmlu
metrics: name: Average accuracy
type: accuracy
value: 27.22
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_winogrande
type: OpenLLM-Ro/ro_winogrande
metrics: name: Average accuracy
type: accuracy
value: 59.29
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_hellaswag
type: OpenLLM-Ro/ro_hellaswag
metrics: name: Average accuracy
type: accuracy
value: 57.22
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_gsm8k
type: OpenLLM-Ro/ro_gsm8k
metrics: name: Average accuracy
type: accuracy
value: 2.53
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_truthfulqa
type: OpenLLM-Ro/ro_truthfulqa
metrics: name: Average accuracy
type: accuracy
value: 44
verified: no, task: type: text-generation
dataset: name: LaRoSeDa_binary
type: LaRoSeDa_binary
metrics: name: Average macro-f1
type: macro-f1
value: 83.25
verified: no, task: type: text-generation
dataset: name: LaRoSeDa_multiclass
type: LaRoSeDa_multiclass
metrics: name: Average macro-f1
type: macro-f1
value: 61.04
verified: no, task: type: text-generation
dataset: name: LaRoSeDa_binary_finetuned
type: LaRoSeDa_binary_finetuned
metrics: name: Average macro-f1
type: macro-f1
value: 98.97
verified: no, task: type: text-generation
dataset: name: LaRoSeDa_multiclass_finetuned
type: LaRoSeDa_multiclass_finetuned
metrics: name: Average macro-f1
type: macro-f1
value: 87.72
verified: no, task: type: text-generation
dataset: name: WMT_EN-RO
type: WMT_EN-RO
metrics: name: Average bleu
type: bleu
value: 10.01
verified: no, task: type: text-generation
dataset: name: WMT_RO-EN
type: WMT_RO-EN
metrics: name: Average bleu
type: bleu
value: 13.03
verified: no, task: type: text-generation
dataset: name: WMT_EN-RO_finetuned
type: WMT_EN-RO_finetuned
metrics: name: Average bleu
type: bleu
value: 27.85
verified: no, task: type: text-generation
dataset: name: WMT_RO-EN_finetuned
type: WMT_RO-EN_finetuned
metrics: name: Average bleu
type: bleu
value: 39.3
verified: no, task: type: text-generation
dataset: name: XQuAD
type: XQuAD
metrics: name: Average exact_match
type: exact_match
value: 30.15
verified: no, task: type: text-generation
dataset: name: XQuAD
type: XQuAD
metrics: name: Average f1
type: f1
value: 47.03
verified: no, task: type: text-generation
dataset: name: XQuAD_finetuned
type: XQuAD_finetuned
metrics: name: Average exact_match
type: exact_match
value: 67.06
verified: no, task: type: text-generation
dataset: name: XQuAD_finetuned
type: XQuAD_finetuned
metrics: name: Average f1
type: f1
value: 79.96
verified: no, task: type: text-generation
dataset: name: STS
type: STS
metrics: name: Average spearman
type: spearman
value: 7.89
verified: no, task: type: text-generation
dataset: name: STS
type: STS
metrics: name: Average pearson
type: pearson
value: 7.98
verified: no, task: type: text-generation
dataset: name: STS_finetuned
type: STS_finetuned
metrics: name: Average spearman
type: spearman
value: 71.75
verified: no, task: type: text-generation
dataset: name: STS_finetuned
type: STS_finetuned
metrics: name: Average pearson
type: pearson
value: 71.99
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_arc_challenge
type: OpenLLM-Ro/ro_arc_challenge
metrics: name: 0-shot
type: accuracy
value: 35.56
verified: no, name: 1-shot
type: accuracy
value: 36.42
verified: no, name: 3-shot
type: accuracy
value: 38.56
verified: no, name: 5-shot
type: accuracy
value: 38.39
verified: no, name: 10-shot
type: accuracy
value: 39.07
verified: no, name: 25-shot
type: accuracy
value: 39.67
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_mmlu
type: OpenLLM-Ro/ro_mmlu
metrics: name: 0-shot
type: accuracy
value: 25.82
verified: no, name: 1-shot
type: accuracy
value: 25.48
verified: no, name: 3-shot
type: accuracy
value: 27.61
verified: no, name: 5-shot
type: accuracy
value: 29.96
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_winogrande
type: OpenLLM-Ro/ro_winogrande
metrics: name: 0-shot
type: accuracy
value: 58.72
verified: no, name: 1-shot
type: accuracy
value: 58.88
verified: no, name: 3-shot
type: accuracy
value: 60.38
verified: no, name: 5-shot
type: accuracy
value: 59.19
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_hellaswag
type: OpenLLM-Ro/ro_hellaswag
metrics: name: 0-shot
type: accuracy
value: 55.85
verified: no, name: 1-shot
type: accuracy
value: 57.06
verified: no, name: 3-shot
type: accuracy
value: 57.52
verified: no, name: 5-shot
type: accuracy
value: 57.89
verified: no, name: 10-shot
type: accuracy
value: 57.79
verified: no, task: type: text-generation
dataset: name: OpenLLM-Ro/ro_gsm8k
type: OpenLLM-Ro/ro_gsm8k
metrics: name: 1-shot
type: accuracy
value: 0
verified: no, name: 3-shot
type: accuracy
value: 2.96
verified: no, name: 5-shot
type: accuracy
value: 4.62
verified: no, task: type: text-generation
dataset: name: LaRoSeDa_binary
type: LaRoSeDa_binary
metrics: name: 0-shot
type: macro-f1
value: 42.78
verified: no, name: 1-shot
type: macro-f1
value: 98
verified: no, name: 3-shot
type: macro-f1
value: 95.13
verified: no, name: 5-shot
type: macro-f1
value: 97.07
verified: no, task: type: text-generation
dataset: name: LaRoSeDa_multiclass
type: LaRoSeDa_multiclass
metrics: name: 0-shot
type: macro-f1
value: 46.41
verified: no, name: 1-shot
type: macro-f1
value: 67.36
verified: no, name: 3-shot
type: macro-f1
value: 65.16
verified: no, name: 5-shot
type: macro-f1
value: 65.23
verified: no, task: type: text-generation
dataset: name: WMT_EN-RO
type: WMT_EN-RO
metrics: name: 0-shot
type: bleu
value: 4.45
verified: no, name: 1-shot
type: bleu
value: 8.61
verified: no, name: 3-shot
type: bleu
value: 12.25
verified: no, name: 5-shot
type: bleu
value: 14.73
verified: no, task: type: text-generation
dataset: name: WMT_RO-EN
type: WMT_RO-EN
metrics: name: 0-shot
type: bleu
value: 1.29
verified: no, name: 1-shot
type: bleu
value: 10.78
verified: no, name: 3-shot
type: bleu
value: 16.82
verified: no, name: 5-shot
type: bleu
value: 23.24
verified: no, task: type: text-generation
dataset: name: XQuAD_EM
type: XQuAD_EM
metrics: name: 0-shot
type: exact_match
value: 5.29
verified: no, name: 1-shot
type: exact_match
value: 33.95
verified: no, name: 3-shot
type: exact_match
value: 39.24
verified: no, name: 5-shot
type: exact_match
value: 42.1
verified: no, task: type: text-generation
dataset: name: XQuAD_F1
type: XQuAD_F1
metrics: name: 0-shot
type: f1
value: 16.17
verified: no, name: 1-shot
type: f1
value: 51.84
verified: no, name: 3-shot
type: f1
value: 58.82
verified: no, name: 5-shot
type: f1
value: 61.29
verified: no, task: type: text-generation
dataset: name: STS_Spearman
type: STS_Spearman
metrics: name: 1-shot
type: spearman
value: -1.74
verified: no, name: 3-shot
type: spearman
value: 15.47
verified: no, name: 5-shot
type: spearman
value: 9.93
verified: no, task: type: text-generation
dataset: name: STS_Pearson
type: STS_Pearson
metrics: name: 1-shot
type: pearson
value: -1.4
verified: no, name: 3-shot
type: pearson
value: 15
verified: no, name: 5-shot
type: pearson
value: 10.33
verified: no
Likes: 4 | All-time downloads: 3788
Created: 2024-02-21 10:16:52