Text Generation
Transformers
Safetensors
English
llama
smol_llama
llama2
Inference Endpoints
text-generation-inference
pszemraj's picture
Upload 5 files
d54c12f
{
"results": {
"arc_easy": {
"acc": 0.41624579124579125,
"acc_stderr": 0.010114819404500878,
"acc_norm": 0.38846801346801346,
"acc_norm_stderr": 0.01000127604448523
},
"boolq": {
"acc": 0.5831804281345566,
"acc_stderr": 0.00862319210884368
},
"lambada_openai": {
"ppl": 79.45218123817662,
"ppl_stderr": 3.1355336623454866,
"acc": 0.2522802251115855,
"acc_stderr": 0.006050943684570117
},
"openbookqa": {
"acc": 0.154,
"acc_stderr": 0.016158285192455334,
"acc_norm": 0.278,
"acc_norm_stderr": 0.02005583388807091
},
"piqa": {
"acc": 0.6050054406964092,
"acc_stderr": 0.011405665187969021,
"acc_norm": 0.5897714907508161,
"acc_norm_stderr": 0.011476256036359109
},
"winogrande": {
"acc": 0.5272296764009471,
"acc_stderr": 0.014031631629827708
}
},
"versions": {
"arc_easy": 0,
"boolq": 1,
"lambada_openai": 0,
"openbookqa": 0,
"piqa": 0,
"winogrande": 0
},
"config": {
"model": "hf-causal-experimental",
"model_args": "pretrained=BEE-spoke-data/smol_llama-81M-tied,revision=main,trust_remote_code=True,dtype='float'",
"num_fewshot": 0,
"batch_size": "16",
"batch_sizes": [],
"device": "cuda",
"no_cache": false,
"limit": null,
"bootstrap_iters": 100000,
"description_dict": {}
}
}