| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.7675016313639117, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7404488937353225, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7658521041744508, | |
| 1.4020329449305176e-289 | |
| ], | |
| "spearman": [ | |
| 0.7730453513050343, | |
| 2.066561283121993e-298 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7623056519318484, | |
| 1.8448021245198884e-262 | |
| ], | |
| "spearman": [ | |
| 0.7572290027800557, | |
| 5.3120116722114526e-257 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7647570029076096, | |
| "mean": 0.7652197958234037, | |
| "wmean": 0.7663843913985655 | |
| }, | |
| "spearman": { | |
| "all": 0.7534738635333297, | |
| "mean": 0.7569077492734708, | |
| "wmean": 0.7487978108339844 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7707517065205873, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6949522587450165, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.778349180191507, | |
| 9.711324138536703e-103 | |
| ], | |
| "spearman": [ | |
| 0.7176593438481393, | |
| 2.743332620783351e-80 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7642987657770804, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.6847455515560564, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7678373187987577, | |
| "mean": 0.7711332174963915, | |
| "wmean": 0.7679316297393041 | |
| }, | |
| "spearman": { | |
| "all": 0.6909477744903625, | |
| "mean": 0.6991190513830707, | |
| "wmean": 0.6910301368785469 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 81.19, | |
| "acc": 80.75, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 87.12, | |
| "acc": 86.73, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.61, | |
| "acc": 95.29, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 89.04, | |
| "acc": 88.61, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 85.21, | |
| "acc": 85.61, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 80.69, | |
| "acc": 86.8, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 71.69, | |
| "acc": 70.32, | |
| "f1": 79.57, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.4514113169306324, | |
| 6.251817668045887e-39 | |
| ], | |
| "spearman": [ | |
| 0.4499549654577979, | |
| 1.1613312893486279e-38 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.8599559900295872, | |
| 1.0643859926359198e-220 | |
| ], | |
| "spearman": [ | |
| 0.8649974875962446, | |
| 3.229629761923383e-226 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.4931333562119396, | |
| 1.6812261260529903e-29 | |
| ], | |
| "spearman": [ | |
| 0.6128103790153274, | |
| 1.1381831791276455e-48 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.7365214189654901, | |
| 3.947242882144461e-129 | |
| ], | |
| "spearman": [ | |
| 0.683719802480883, | |
| 1.910512439440496e-104 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.6084967943558258, | |
| 8.882847362372895e-42 | |
| ], | |
| "spearman": [ | |
| 0.6324009475046439, | |
| 5.9238371155257115e-46 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6090786981125251, | |
| "mean": 0.629903775298695, | |
| "wmean": 0.6451270836208292 | |
| }, | |
| "spearman": { | |
| "all": 0.5915463354166586, | |
| "mean": 0.6487767164109794, | |
| "wmean": 0.6539936080030831 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.5827724320641434, | |
| 1.399853624093483e-18 | |
| ], | |
| "spearman": [ | |
| 0.6243224304995846, | |
| 8.1205413287147755e-22 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.782141211640678, | |
| 7.962032809197313e-156 | |
| ], | |
| "spearman": [ | |
| 0.7864730688106963, | |
| 1.0734436155388987e-158 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.7763285115488442, | |
| 3.939259823816713e-114 | |
| ], | |
| "spearman": [ | |
| 0.7706743357210359, | |
| 1.7455185991511694e-111 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7269930269261002, | |
| "mean": 0.7137473850845552, | |
| "wmean": 0.7548467955796887 | |
| }, | |
| "spearman": { | |
| "all": 0.7392805868780532, | |
| "mean": 0.7271566116771057, | |
| "wmean": 0.7601333622079632 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.5228897751421426, | |
| 6.073682402382636e-33 | |
| ], | |
| "spearman": [ | |
| 0.5053330319071232, | |
| 1.532862819818703e-30 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.7680069983657648, | |
| 1.2852042740473546e-59 | |
| ], | |
| "spearman": [ | |
| 0.7189884605027314, | |
| 5.271976030712246e-49 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7541282525177266, | |
| 9.793324818317855e-139 | |
| ], | |
| "spearman": [ | |
| 0.7184718603499471, | |
| 4.702376618230486e-120 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8478349220740108, | |
| 2.8453760358482384e-208 | |
| ], | |
| "spearman": [ | |
| 0.8081484798616755, | |
| 3.9047867377054607e-174 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8215837025605404, | |
| 9.871899662708195e-185 | |
| ], | |
| "spearman": [ | |
| 0.8279072975227956, | |
| 4.92735009919912e-190 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.7317126218269044, | |
| 1.2202543536716308e-126 | |
| ], | |
| "spearman": [ | |
| 0.6520317484632859, | |
| 5.3026345548268765e-92 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7189696594882792, | |
| "mean": 0.7410260454145149, | |
| "wmean": 0.7552392326821546 | |
| }, | |
| "spearman": { | |
| "all": 0.670800547509326, | |
| "mean": 0.7051468131012597, | |
| "wmean": 0.7194709179086142 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.7262407045308339, | |
| 1.1025031075653589e-62 | |
| ], | |
| "spearman": [ | |
| 0.7275038094083748, | |
| 5.324790051370908e-63 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.6978521677757564, | |
| 1.6036590447498983e-110 | |
| ], | |
| "spearman": [ | |
| 0.7024365108998977, | |
| 1.4353461268590496e-112 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.7839138106441258, | |
| 3.235687910099277e-79 | |
| ], | |
| "spearman": [ | |
| 0.79336254174086, | |
| 2.047534842473332e-82 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.8124656869189333, | |
| 1.9047661503782033e-177 | |
| ], | |
| "spearman": [ | |
| 0.8124179763234277, | |
| 2.0744785143960583e-177 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8731830765774482, | |
| 1.1398312421407709e-235 | |
| ], | |
| "spearman": [ | |
| 0.8795017881810903, | |
| 1.9914488519259588e-243 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7827019127153672, | |
| "mean": 0.7787310892894196, | |
| "wmean": 0.7846445472149044 | |
| }, | |
| "spearman": { | |
| "all": 0.790296617367925, | |
| "mean": 0.7830445253107301, | |
| "wmean": 0.7886973627447582 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.7325081931498698, | |
| 5.607126931548686e-44 | |
| ], | |
| "spearman": [ | |
| 0.7313808706110454, | |
| 8.788498983014453e-44 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7702918518354713, | |
| 3.5986232616752304e-50 | |
| ], | |
| "spearman": [ | |
| 0.7838189633800285, | |
| 5.030858666324319e-53 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.8193777086614761, | |
| 5.14320199251657e-57 | |
| ], | |
| "spearman": [ | |
| 0.8274487763356487, | |
| 4.601518050831221e-59 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.8373109699109689, | |
| 2.1202879642927884e-65 | |
| ], | |
| "spearman": [ | |
| 0.8554383865248261, | |
| 4.218785247850647e-71 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7703959894794563, | |
| 2.4658565942110764e-42 | |
| ], | |
| "spearman": [ | |
| 0.7772827176027448, | |
| 1.5625377662687375e-43 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7784793728977035, | |
| "mean": 0.7859769426074485, | |
| "wmean": 0.785525517385094 | |
| }, | |
| "spearman": { | |
| "all": 0.7906179855205325, | |
| "mean": 0.7950739428908588, | |
| "wmean": 0.7946323236636283 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.5915463354166586, | |
| "eval_senteval-STS13": 0.7392805868780532, | |
| "eval_senteval-STS14": 0.670800547509326, | |
| "eval_senteval-STS15": 0.790296617367925, | |
| "eval_senteval-STS16": 0.7906179855205325, | |
| "eval_senteval-STSBenchmark": 0.7534738635333297, | |
| "eval_senteval-SICKRelatedness": 0.6909477744903625, | |
| "eval_senteval-avg_sts_7": 0.7181376729594552, | |
| "eval_senteval-MR": 81.19, | |
| "eval_senteval-CR": 87.12, | |
| "eval_senteval-SUBJ": 95.61, | |
| "eval_senteval-MPQA": 89.04, | |
| "eval_senteval-SST2": 85.21, | |
| "eval_senteval-TREC": 80.69, | |
| "eval_senteval-MRPC": 71.69, | |
| "eval_senteval-avg_transfer": 84.36428571428571 | |
| } | |