{ "STSBenchmark": { "train": { "pearson": [ 0.7078537910606651, 0.0 ], "spearman": [ 0.7005999583866984, 0.0 ], "nsamples": 5749 }, "dev": { "pearson": [ 0.7192800747133679, 3.077700364928066e-239 ], "spearman": [ 0.7379237505860065, 4.277474410967731e-258 ], "nsamples": 1500 }, "test": { "pearson": [ 0.6983884458054606, 2.940291533445568e-202 ], "spearman": [ 0.7136101793848104, 4.3889157567246634e-215 ], "nsamples": 1379 }, "all": { "pearson": { "all": 0.7084868825859301, "mean": 0.7085074371931644, "wmean": 0.7083274482665213 }, "spearman": { "all": 0.7134156978222834, "mean": 0.7173779627858385, "wmean": 0.7091681993527809 } } }, "SICKRelatedness": { "train": { "pearson": [ 0.7474648285643389, 0.0 ], "spearman": [ 0.6940199688267912, 0.0 ], "nsamples": 4500 }, "dev": { "pearson": [ 0.7666627901165247, 6.873435561530908e-98 ], "spearman": [ 0.7253075282545358, 8.79855875263332e-83 ], "nsamples": 500 }, "test": { "pearson": [ 0.7471456909275643, 0.0 ], "spearman": [ 0.6877954280401497, 0.0 ], "nsamples": 4927 }, "all": { "pearson": { "all": 0.7481870904024901, "mean": 0.753757769869476, "wmean": 0.7482733900269867 }, "spearman": { "all": 0.6925290217821645, "mean": 0.7023743083738255, "wmean": 0.6925064669891857 } } }, "MR": { "devacc": 82.2, "acc": 81.86, "ndev": 10662, "ntest": 10662 }, "CR": { "devacc": 88.38, "acc": 87.87, "ndev": 3775, "ntest": 3775 }, "SUBJ": { "devacc": 95.78, "acc": 95.45, "ndev": 10000, "ntest": 10000 }, "MPQA": { "devacc": 89.44, "acc": 89.26, "ndev": 10606, "ntest": 10606 }, "SST2": { "devacc": 84.29, "acc": 85.83, "ndev": 872, "ntest": 1821 }, "TREC": { "devacc": 80.47, "acc": 88.2, "ndev": 5452, "ntest": 500 }, "MRPC": { "devacc": 70.61, "acc": 69.28, "f1": 80.28, "ndev": 4076, "ntest": 1725 }, "STS12": { "MSRpar": { "pearson": [ 0.4308787229359875, 2.951518773589667e-35 ], "spearman": [ 0.43588907674191796, 3.949785529584259e-36 ], "nsamples": 750 }, "MSRvid": { "pearson": [ 0.7685772608624017, 3.0060842440727446e-147 ], "spearman": [ 0.7784022398247612, 2.1180197452085128e-153 ], "nsamples": 750 }, "SMTeuroparl": { "pearson": [ 0.48420310815012313, 2.3512310040226806e-28 ], "spearman": [ 0.5830380352830036, 3.802237461067692e-43 ], "nsamples": 459 }, "surprise.OnWN": { "pearson": [ 0.7022395609517124, 1.7609879155318726e-112 ], "spearman": [ 0.6542675353808592, 7.864813884891985e-93 ], "nsamples": 750 }, "surprise.SMTnews": { "pearson": [ 0.5667702919910472, 2.7779222845650156e-35 ], "spearman": [ 0.6138135967602001, 1.1234322018162595e-42 ], "nsamples": 399 }, "all": { "pearson": { "all": 0.5248106860340233, "mean": 0.5905337889782544, "wmean": 0.6031731762251964 }, "spearman": { "all": 0.551472253212969, "mean": 0.6130820967981483, "wmean": 0.6158124910755702 } } }, "STS13": { "FNWN": { "pearson": [ 0.5506969449768325, 2.2186867879566377e-16 ], "spearman": [ 0.5845168173317136, 1.0457395906514476e-18 ], "nsamples": 189 }, "headlines": { "pearson": [ 0.7701375188545448, 3.323272248926893e-148 ], "spearman": [ 0.7914045646334811, 4.792158949648843e-162 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.7396168929190757, 3.380330666101667e-98 ], "spearman": [ 0.7366350658057337, 5.06557475163485e-97 ], "nsamples": 561 }, "all": { "pearson": { "all": 0.6820859764553678, "mean": 0.6868171189168177, "wmean": 0.7310732924460877 }, "spearman": { "all": 0.7229160162899463, "mean": 0.7041854825903094, "wmean": 0.7448529159118809 } } }, "STS14": { "deft-forum": { "pearson": [ 0.4362756518936318, 2.4689546739502166e-22 ], "spearman": [ 0.4356265722089346, 2.891472829972546e-22 ], "nsamples": 450 }, "deft-news": { "pearson": [ 0.7383916191185764, 6.413175758096686e-53 ], "spearman": [ 0.7137105870085976, 5.373218095899081e-48 ], "nsamples": 300 }, "headlines": { "pearson": [ 0.7468974428675955, 1.0765636396917894e-134 ], "spearman": [ 0.7245056476016037, 5.235979722924423e-123 ], "nsamples": 750 }, "images": { "pearson": [ 0.796822057327234, 7.832124393347635e-166 ], "spearman": [ 0.7650434028457997, 4.13659539181102e-145 ], "nsamples": 750 }, "OnWN": { "pearson": [ 0.7857012591480514, 3.5241880397200474e-158 ], "spearman": [ 0.7967713894609839, 8.507992228801468e-166 ], "nsamples": 750 }, "tweet-news": { "pearson": [ 0.6595728752282025, 7.952319515520837e-95 ], "spearman": [ 0.6114329574279981, 4.316968711309796e-78 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.6649722439404772, "mean": 0.6939434842638819, "wmean": 0.7092231346709386 }, "spearman": { "all": 0.6332541978552203, "mean": 0.6745150927589862, "wmean": 0.688922715093037 } } }, "STS15": { "answers-forums": { "pearson": [ 0.6550074272217491, 2.5452363132722557e-47 ], "spearman": [ 0.6570961934421593, 1.034283039715521e-47 ], "nsamples": 375 }, "answers-students": { "pearson": [ 0.6832984945676881, 2.8640939149258605e-104 ], "spearman": [ 0.6907992077474371, 1.9110197999110346e-107 ], "nsamples": 750 }, "belief": { "pearson": [ 0.6709117751011916, 2.2248924681984537e-50 ], "spearman": [ 0.6943992418065011, 2.9220535633271436e-55 ], "nsamples": 375 }, "headlines": { "pearson": [ 0.8086773826726481, 1.5503098286475848e-174 ], "spearman": [ 0.8205804721828502, 6.549512205685779e-184 ], "nsamples": 750 }, "images": { "pearson": [ 0.8326574715302321, 3.6753187356312923e-194 ], "spearman": [ 0.8494884877285356, 6.66575077788996e-210 ], "nsamples": 750 }, "all": { "pearson": { "all": 0.7334203057310366, "mean": 0.7301105102187018, "wmean": 0.7468982374830097 }, "spearman": { "all": 0.7527302594998441, "mean": 0.7424727205814967, "wmean": 0.7591539713207883 } } }, "STS16": { "answer-answer": { "pearson": [ 0.6774976875587965, 1.7801175736052472e-35 ], "spearman": [ 0.6996537261374689, 1.1481023213168017e-38 ], "nsamples": 254 }, "headlines": { "pearson": [ 0.7723091430702096, 1.3899358276799273e-50 ], "spearman": [ 0.797440407232274, 4.088502753480497e-56 ], "nsamples": 249 }, "plagiarism": { "pearson": [ 0.8114637165686283, 4.19426284677851e-55 ], "spearman": [ 0.8340872996890834, 7.881725319141058e-61 ], "nsamples": 230 }, "postediting": { "pearson": [ 0.7526664902410026, 8.06271051079258e-46 ], "spearman": [ 0.7786484784235972, 6.807428083289534e-51 ], "nsamples": 244 }, "question-question": { "pearson": [ 0.7694288722010691, 3.6047724274094633e-42 ], "spearman": [ 0.785768908279152, 4.540569131264119e-45 ], "nsamples": 209 }, "all": { "pearson": { "all": 0.7474943176404166, "mean": 0.7566731819279412, "wmean": 0.7550483153322336 }, "spearman": { "all": 0.7752096236911921, "mean": 0.7791197639523151, "wmean": 0.7776818864535776 } } }, "eval_senteval-STS12": 0.551472253212969, "eval_senteval-STS13": 0.7229160162899463, "eval_senteval-STS14": 0.6332541978552203, "eval_senteval-STS15": 0.7527302594998441, "eval_senteval-STS16": 0.7752096236911921, "eval_senteval-STSBenchmark": 0.7134156978222834, "eval_senteval-SICKRelatedness": 0.6925290217821645, "eval_senteval-avg_sts_7": 0.6916467243076598, "eval_senteval-MR": 82.2, "eval_senteval-CR": 88.38, "eval_senteval-SUBJ": 95.78, "eval_senteval-MPQA": 89.44, "eval_senteval-SST2": 84.29, "eval_senteval-TREC": 80.47, "eval_senteval-MRPC": 70.61, "eval_senteval-avg_transfer": 84.45285714285716 }