Update README.md
Browse files
README.md
CHANGED
|
@@ -20,3 +20,49 @@ Fully reproducible finetuning code is available on [GitHub](https://github.com/l
|
|
| 20 |
|
| 21 |
See [Pythia-1.4b](https://huggingface.co/EleutherAI/pythia-1.4b) for model details [(paper)](https://arxiv.org/abs/2101.00027).
|
| 22 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
See [Pythia-1.4b](https://huggingface.co/EleutherAI/pythia-1.4b) for model details [(paper)](https://arxiv.org/abs/2101.00027).
|
| 22 |
|
| 23 |
+
hf (pretrained=lomahony/pythia-1.4b-helpful-sft), gen_kwargs: (None), limit: None, num_fewshot: 0, batch_size: 16
|
| 24 |
+
| Tasks |Version|Filter|n-shot| Metric | Value | |Stderr|
|
| 25 |
+
|--------------|------:|------|-----:|---------------|------:|---|------|
|
| 26 |
+
|arc_challenge | 1|none | 0|acc | 0.2679|± |0.0129|
|
| 27 |
+
| | |none | 0|acc_norm | 0.2978|± |0.0134|
|
| 28 |
+
|arc_easy | 1|none | 0|acc | 0.6120|± |0.0100|
|
| 29 |
+
| | |none | 0|acc_norm | 0.5282|± |0.0102|
|
| 30 |
+
|boolq | 2|none | 0|acc | 0.6260|± |0.0085|
|
| 31 |
+
|hellaswag | 1|none | 0|acc | 0.4097|± |0.0049|
|
| 32 |
+
| | |none | 0|acc_norm | 0.5212|± |0.0050|
|
| 33 |
+
|lambada_openai| 1|none | 0|perplexity | 6.4836|± |0.1838|
|
| 34 |
+
| | |none | 0|acc | 0.5789|± |0.0069|
|
| 35 |
+
|openbookqa | 1|none | 0|acc | 0.2120|± |0.0183|
|
| 36 |
+
| | |none | 0|acc_norm | 0.3340|± |0.0211|
|
| 37 |
+
|piqa | 1|none | 0|acc | 0.7100|± |0.0106|
|
| 38 |
+
| | |none | 0|acc_norm | 0.7144|± |0.0105|
|
| 39 |
+
|sciq | 1|none | 0|acc | 0.8540|± |0.0112|
|
| 40 |
+
| | |none | 0|acc_norm | 0.7830|± |0.0130|
|
| 41 |
+
|wikitext | 2|none | 0|word_perplexity|15.8394|± |N/A |
|
| 42 |
+
| | |none | 0|byte_perplexity| 1.6763|± |N/A |
|
| 43 |
+
| | |none | 0|bits_per_byte | 0.7453|± |N/A |
|
| 44 |
+
|winogrande | 1|none | 0|acc | 0.5872|± |0.0138|
|
| 45 |
+
|
| 46 |
+
hf (pretrained=lomahony/pythia-1.4b-helpful-sft), gen_kwargs: (None), limit: None, num_fewshot: 5, batch_size: 16
|
| 47 |
+
| Tasks |Version|Filter|n-shot| Metric | Value | |Stderr|
|
| 48 |
+
|--------------|------:|------|-----:|---------------|------:|---|------|
|
| 49 |
+
|arc_challenge | 1|none | 5|acc | 0.2892|± |0.0133|
|
| 50 |
+
| | |none | 5|acc_norm | 0.3097|± |0.0135|
|
| 51 |
+
|arc_easy | 1|none | 5|acc | 0.6444|± |0.0098|
|
| 52 |
+
| | |none | 5|acc_norm | 0.6309|± |0.0099|
|
| 53 |
+
|boolq | 2|none | 5|acc | 0.6333|± |0.0084|
|
| 54 |
+
|hellaswag | 1|none | 5|acc | 0.4065|± |0.0049|
|
| 55 |
+
| | |none | 5|acc_norm | 0.5215|± |0.0050|
|
| 56 |
+
|lambada_openai| 1|none | 5|perplexity | 9.7040|± |0.2887|
|
| 57 |
+
| | |none | 5|acc | 0.4951|± |0.0070|
|
| 58 |
+
|openbookqa | 1|none | 5|acc | 0.2220|± |0.0186|
|
| 59 |
+
| | |none | 5|acc_norm | 0.3100|± |0.0207|
|
| 60 |
+
|piqa | 1|none | 5|acc | 0.7029|± |0.0107|
|
| 61 |
+
| | |none | 5|acc_norm | 0.7127|± |0.0106|
|
| 62 |
+
|sciq | 1|none | 5|acc | 0.9170|± |0.0087|
|
| 63 |
+
| | |none | 5|acc_norm | 0.9160|± |0.0088|
|
| 64 |
+
|wikitext | 2|none | 5|word_perplexity|15.8394|± |N/A |
|
| 65 |
+
| | |none | 5|byte_perplexity| 1.6763|± |N/A |
|
| 66 |
+
| | |none | 5|bits_per_byte | 0.7453|± |N/A |
|
| 67 |
+
|winogrande | 1|none | 5|acc | 0.5699|± |0.0139|
|
| 68 |
+
|