--- license: mit # tags: # - mteb model-index: - name: hui-embedding results: - task: type: STS dataset: type: C-MTEB/AFQMC name: MTEB AFQMC config: default split: validation revision: b44c3b011063adb25877c13823db83bb193913c4 metrics: - type: cos_sim_pearson value: 49.378812776509605 - type: cos_sim_spearman value: 54.847928348721375 - type: euclidean_pearson value: 53.04925208393059 - type: euclidean_spearman value: 54.847928348721375 - type: manhattan_pearson value: 53.06246682259037 - type: manhattan_spearman value: 54.87201329143453 - task: type: STS dataset: type: C-MTEB/ATEC name: MTEB ATEC config: default split: test revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865 metrics: - type: cos_sim_pearson value: 48.1600594371303 - type: cos_sim_spearman value: 55.1323822895328 - type: euclidean_pearson value: 55.4361186282293 - type: euclidean_spearman value: 55.13238454056627 - type: manhattan_pearson value: 55.41462879819177 - type: manhattan_spearman value: 55.13393260953372 - task: type: Classification dataset: type: mteb/amazon_reviews_multi name: MTEB AmazonReviewsClassification (zh) config: zh split: test revision: 1399c76144fd37290681b995c656ef9b2e06e26d metrics: - type: accuracy value: 46.715999999999994 - type: f1 value: 45.031952047233084 - task: type: STS dataset: type: C-MTEB/BQ name: MTEB BQ config: default split: test revision: e3dda5e115e487b39ec7e618c0c6a29137052a55 metrics: - type: cos_sim_pearson value: 63.51780001831422 - type: cos_sim_spearman value: 65.57016807800922 - type: euclidean_pearson value: 64.0519397443374 - type: euclidean_spearman value: 65.57018193343218 - type: manhattan_pearson value: 63.95723624728276 - type: manhattan_spearman value: 65.48557125105762 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringP2P name: MTEB CLSClusteringP2P config: default split: test revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476 metrics: - type: v_measure value: 42.5667507726515 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringS2S name: MTEB CLSClusteringS2S config: default split: test revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f metrics: - type: v_measure value: 40.510799373442566 - task: type: Reranking dataset: type: C-MTEB/CMedQAv1-reranking name: MTEB CMedQAv1 config: default split: test revision: 8d7f1e942507dac42dc58017c1a001c3717da7df metrics: - type: map value: 87.12221041649349 - type: mrr value: 89.57150793650794 - task: type: Reranking dataset: type: C-MTEB/CMedQAv2-reranking name: MTEB CMedQAv2 config: default split: test revision: 23d186750531a14a0357ca22cd92d712fd512ea0 metrics: - type: map value: 87.59790061886422 - type: mrr value: 90.19202380952382 - task: type: Retrieval dataset: type: C-MTEB/CmedqaRetrieval name: MTEB CmedqaRetrieval config: default split: dev revision: cd540c506dae1cf9e9a59c3e06f42030d54e7301 metrics: - type: map_at_1 value: 25.22 - type: map_at_10 value: 37.611 - type: map_at_100 value: 39.505 - type: map_at_1000 value: 39.617999999999995 - type: map_at_3 value: 33.387 - type: map_at_5 value: 35.776 - type: mrr_at_1 value: 38.385000000000005 - type: mrr_at_10 value: 46.487 - type: mrr_at_100 value: 47.508 - type: mrr_at_1000 value: 47.55 - type: mrr_at_3 value: 43.897999999999996 - type: mrr_at_5 value: 45.371 - type: ndcg_at_1 value: 38.385000000000005 - type: ndcg_at_10 value: 44.232 - type: ndcg_at_100 value: 51.639 - type: ndcg_at_1000 value: 53.561 - type: ndcg_at_3 value: 38.855000000000004 - type: ndcg_at_5 value: 41.159 - type: precision_at_1 value: 38.385000000000005 - type: precision_at_10 value: 9.817 - type: precision_at_100 value: 1.58 - type: precision_at_1000 value: 0.183 - type: precision_at_3 value: 21.88 - type: precision_at_5 value: 15.969 - type: recall_at_1 value: 25.22 - type: recall_at_10 value: 54.909 - type: recall_at_100 value: 85.469 - type: recall_at_1000 value: 98.18599999999999 - type: recall_at_3 value: 38.834999999999994 - type: recall_at_5 value: 45.87 - task: type: PairClassification dataset: type: C-MTEB/CMNLI name: MTEB Cmnli config: default split: validation revision: 41bc36f332156f7adc9e38f53777c959b2ae9766 metrics: - type: cos_sim_accuracy value: 83.22309079975948 - type: cos_sim_ap value: 89.94908564295963 - type: cos_sim_f1 value: 84.3951165371809 - type: cos_sim_precision value: 80.32960067610395 - type: cos_sim_recall value: 88.89408463876549 - type: dot_accuracy value: 83.22309079975948 - type: dot_ap value: 89.95437246807009 - type: dot_f1 value: 84.3951165371809 - type: dot_precision value: 80.32960067610395 - type: dot_recall value: 88.89408463876549 - type: euclidean_accuracy value: 83.22309079975948 - type: euclidean_ap value: 89.94908838738027 - type: euclidean_f1 value: 84.3951165371809 - type: euclidean_precision value: 80.32960067610395 - type: euclidean_recall value: 88.89408463876549 - type: manhattan_accuracy value: 83.23511725796753 - type: manhattan_ap value: 89.94576222197006 - type: manhattan_f1 value: 84.44100409156253 - type: manhattan_precision value: 80.10910616869492 - type: manhattan_recall value: 89.26817862988075 - type: max_accuracy value: 83.23511725796753 - type: max_ap value: 89.95437246807009 - type: max_f1 value: 84.44100409156253 - task: type: Retrieval dataset: type: C-MTEB/CovidRetrieval name: MTEB CovidRetrieval config: default split: dev revision: 1271c7809071a13532e05f25fb53511ffce77117 metrics: - type: map_at_1 value: 76.87 - type: map_at_10 value: 84.513 - type: map_at_100 value: 84.616 - type: map_at_1000 value: 84.61800000000001 - type: map_at_3 value: 83.127 - type: map_at_5 value: 83.99600000000001 - type: mrr_at_1 value: 77.239 - type: mrr_at_10 value: 84.583 - type: mrr_at_100 value: 84.68 - type: mrr_at_1000 value: 84.681 - type: mrr_at_3 value: 83.281 - type: mrr_at_5 value: 84.097 - type: ndcg_at_1 value: 77.134 - type: ndcg_at_10 value: 87.71600000000001 - type: ndcg_at_100 value: 88.172 - type: ndcg_at_1000 value: 88.211 - type: ndcg_at_3 value: 84.993 - type: ndcg_at_5 value: 86.519 - type: precision_at_1 value: 77.134 - type: precision_at_10 value: 9.852 - type: precision_at_100 value: 1.006 - type: precision_at_1000 value: 0.101 - type: precision_at_3 value: 30.313000000000002 - type: precision_at_5 value: 18.945999999999998 - type: recall_at_1 value: 76.87 - type: recall_at_10 value: 97.524 - type: recall_at_100 value: 99.579 - type: recall_at_1000 value: 99.895 - type: recall_at_3 value: 90.227 - type: recall_at_5 value: 93.888 - task: type: Retrieval dataset: type: C-MTEB/DuRetrieval name: MTEB DuRetrieval config: default split: dev revision: a1a333e290fe30b10f3f56498e3a0d911a693ced metrics: - type: map_at_1 value: 25.941 - type: map_at_10 value: 78.793 - type: map_at_100 value: 81.583 - type: map_at_1000 value: 81.63 - type: map_at_3 value: 54.767999999999994 - type: map_at_5 value: 69.16499999999999 - type: mrr_at_1 value: 90.45 - type: mrr_at_10 value: 93.401 - type: mrr_at_100 value: 93.452 - type: mrr_at_1000 value: 93.45599999999999 - type: mrr_at_3 value: 93.10000000000001 - type: mrr_at_5 value: 93.27499999999999 - type: ndcg_at_1 value: 90.45 - type: ndcg_at_10 value: 86.433 - type: ndcg_at_100 value: 89.287 - type: ndcg_at_1000 value: 89.74600000000001 - type: ndcg_at_3 value: 85.64500000000001 - type: ndcg_at_5 value: 84.444 - type: precision_at_1 value: 90.45 - type: precision_at_10 value: 41.185 - type: precision_at_100 value: 4.761 - type: precision_at_1000 value: 0.48700000000000004 - type: precision_at_3 value: 76.617 - type: precision_at_5 value: 64.68 - type: recall_at_1 value: 25.941 - type: recall_at_10 value: 87.393 - type: recall_at_100 value: 96.54 - type: recall_at_1000 value: 98.956 - type: recall_at_3 value: 56.976000000000006 - type: recall_at_5 value: 73.714 - task: type: Retrieval dataset: type: C-MTEB/EcomRetrieval name: MTEB EcomRetrieval config: default split: dev revision: 687de13dc7294d6fd9be10c6945f9e8fec8166b9 metrics: - type: map_at_1 value: 52.900000000000006 - type: map_at_10 value: 63.139 - type: map_at_100 value: 63.62799999999999 - type: map_at_1000 value: 63.639 - type: map_at_3 value: 60.817 - type: map_at_5 value: 62.197 - type: mrr_at_1 value: 52.900000000000006 - type: mrr_at_10 value: 63.139 - type: mrr_at_100 value: 63.62799999999999 - type: mrr_at_1000 value: 63.639 - type: mrr_at_3 value: 60.817 - type: mrr_at_5 value: 62.197 - type: ndcg_at_1 value: 52.900000000000006 - type: ndcg_at_10 value: 68.038 - type: ndcg_at_100 value: 70.41199999999999 - type: ndcg_at_1000 value: 70.717 - type: ndcg_at_3 value: 63.287000000000006 - type: ndcg_at_5 value: 65.766 - type: precision_at_1 value: 52.900000000000006 - type: precision_at_10 value: 8.34 - type: precision_at_100 value: 0.9450000000000001 - type: precision_at_1000 value: 0.097 - type: precision_at_3 value: 23.467 - type: precision_at_5 value: 15.28 - type: recall_at_1 value: 52.900000000000006 - type: recall_at_10 value: 83.39999999999999 - type: recall_at_100 value: 94.5 - type: recall_at_1000 value: 96.89999999999999 - type: recall_at_3 value: 70.39999999999999 - type: recall_at_5 value: 76.4 - task: type: Classification dataset: type: C-MTEB/IFlyTek-classification name: MTEB IFlyTek config: default split: validation revision: 421605374b29664c5fc098418fe20ada9bd55f8a metrics: - type: accuracy value: 49.74220854174683 - type: f1 value: 38.01254780588847 - task: type: Classification dataset: type: C-MTEB/JDReview-classification name: MTEB JDReview config: default split: test revision: b7c64bd89eb87f8ded463478346f76731f07bf8b metrics: - type: accuracy value: 86.73545966228893 - type: ap value: 55.72394235169542 - type: f1 value: 81.58550390953492 - task: type: STS dataset: type: C-MTEB/LCQMC name: MTEB LCQMC config: default split: test revision: 17f9b096f80380fce5ed12a9be8be7784b337daf metrics: - type: cos_sim_pearson value: 69.96738688541858 - type: cos_sim_spearman value: 75.54733864118711 - type: euclidean_pearson value: 74.62650665031346 - type: euclidean_spearman value: 75.54734085635552 - type: manhattan_pearson value: 74.60961858490293 - type: manhattan_spearman value: 75.5274896144882 - task: type: Reranking dataset: type: C-MTEB/Mmarco-reranking name: MTEB MMarcoReranking config: default split: dev revision: 8e0c766dbe9e16e1d221116a3f36795fbade07f6 metrics: - type: map value: 28.07658854954957 - type: mrr value: 26.98015873015873 - task: type: Retrieval dataset: type: C-MTEB/MMarcoRetrieval name: MTEB MMarcoRetrieval config: default split: dev revision: 539bbde593d947e2a124ba72651aafc09eb33fc2 metrics: - type: map_at_1 value: 65.566 - type: map_at_10 value: 74.75699999999999 - type: map_at_100 value: 75.071 - type: map_at_1000 value: 75.08500000000001 - type: map_at_3 value: 72.965 - type: map_at_5 value: 74.146 - type: mrr_at_1 value: 67.808 - type: mrr_at_10 value: 75.42999999999999 - type: mrr_at_100 value: 75.696 - type: mrr_at_1000 value: 75.708 - type: mrr_at_3 value: 73.902 - type: mrr_at_5 value: 74.902 - type: ndcg_at_1 value: 67.808 - type: ndcg_at_10 value: 78.469 - type: ndcg_at_100 value: 79.886 - type: ndcg_at_1000 value: 80.257 - type: ndcg_at_3 value: 75.115 - type: ndcg_at_5 value: 77.086 - type: precision_at_1 value: 67.808 - type: precision_at_10 value: 9.474 - type: precision_at_100 value: 1.019 - type: precision_at_1000 value: 0.105 - type: precision_at_3 value: 28.266000000000002 - type: precision_at_5 value: 18.014 - type: recall_at_1 value: 65.566 - type: recall_at_10 value: 89.193 - type: recall_at_100 value: 95.64399999999999 - type: recall_at_1000 value: 98.541 - type: recall_at_3 value: 80.41199999999999 - type: recall_at_5 value: 85.056 - task: type: Classification dataset: type: mteb/amazon_massive_intent name: MTEB MassiveIntentClassification (zh-CN) config: zh-CN split: test revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 metrics: - type: accuracy value: 72.85810356422327 - type: f1 value: 70.24900138264265 - task: type: Classification dataset: type: mteb/amazon_massive_scenario name: MTEB MassiveScenarioClassification (zh-CN) config: zh-CN split: test revision: 7d571f92784cd94a019292a1f45445077d0ef634 metrics: - type: accuracy value: 75.47074646940149 - type: f1 value: 75.02944928791243 - task: type: Retrieval dataset: type: C-MTEB/MedicalRetrieval name: MTEB MedicalRetrieval config: default split: dev revision: 2039188fb5800a9803ba5048df7b76e6fb151fc6 metrics: - type: map_at_1 value: 53.300000000000004 - type: map_at_10 value: 60.086 - type: map_at_100 value: 60.633 - type: map_at_1000 value: 60.67399999999999 - type: map_at_3 value: 58.567 - type: map_at_5 value: 59.44199999999999 - type: mrr_at_1 value: 53.400000000000006 - type: mrr_at_10 value: 60.143 - type: mrr_at_100 value: 60.69799999999999 - type: mrr_at_1000 value: 60.739 - type: mrr_at_3 value: 58.633 - type: mrr_at_5 value: 59.508 - type: ndcg_at_1 value: 53.300000000000004 - type: ndcg_at_10 value: 63.388 - type: ndcg_at_100 value: 66.257 - type: ndcg_at_1000 value: 67.419 - type: ndcg_at_3 value: 60.224 - type: ndcg_at_5 value: 61.794000000000004 - type: precision_at_1 value: 53.300000000000004 - type: precision_at_10 value: 7.380000000000001 - type: precision_at_100 value: 0.877 - type: precision_at_1000 value: 0.097 - type: precision_at_3 value: 21.667 - type: precision_at_5 value: 13.76 - type: recall_at_1 value: 53.300000000000004 - type: recall_at_10 value: 73.8 - type: recall_at_100 value: 87.7 - type: recall_at_1000 value: 97.0 - type: recall_at_3 value: 65.0 - type: recall_at_5 value: 68.8 - task: type: Classification dataset: type: C-MTEB/MultilingualSentiment-classification name: MTEB MultilingualSentiment config: default split: validation revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a metrics: - type: accuracy value: 76.28000000000002 - type: f1 value: 76.31632893766579 - task: type: PairClassification dataset: type: C-MTEB/OCNLI name: MTEB Ocnli config: default split: validation revision: 66e76a618a34d6d565d5538088562851e6daa7ec metrics: - type: cos_sim_accuracy value: 78.72225230102869 - type: cos_sim_ap value: 80.64016185061472 - type: cos_sim_f1 value: 80.52190121155638 - type: cos_sim_precision value: 72.06005004170142 - type: cos_sim_recall value: 91.23548046462513 - type: dot_accuracy value: 78.72225230102869 - type: dot_ap value: 80.64030954198267 - type: dot_f1 value: 80.52190121155638 - type: dot_precision value: 72.06005004170142 - type: dot_recall value: 91.23548046462513 - type: euclidean_accuracy value: 78.72225230102869 - type: euclidean_ap value: 80.64016185061472 - type: euclidean_f1 value: 80.52190121155638 - type: euclidean_precision value: 72.06005004170142 - type: euclidean_recall value: 91.23548046462513 - type: manhattan_accuracy value: 78.18083378451544 - type: manhattan_ap value: 80.52168370151603 - type: manhattan_f1 value: 80.43684710351377 - type: manhattan_precision value: 73.0802415875755 - type: manhattan_recall value: 89.44033790918691 - type: max_accuracy value: 78.72225230102869 - type: max_ap value: 80.64030954198267 - type: max_f1 value: 80.52190121155638 - task: type: Classification dataset: type: C-MTEB/OnlineShopping-classification name: MTEB OnlineShopping config: default split: test revision: e610f2ebd179a8fda30ae534c3878750a96db120 metrics: - type: accuracy value: 92.50999999999999 - type: ap value: 90.6813824215301 - type: f1 value: 92.50242491859119 - task: type: STS dataset: type: C-MTEB/PAWSX name: MTEB PAWSX config: default split: test revision: 9c6a90e430ac22b5779fb019a23e820b11a8b5e1 metrics: - type: cos_sim_pearson value: 33.62875118995766 - type: cos_sim_spearman value: 37.89501991717997 - type: euclidean_pearson value: 38.11011221831966 - type: euclidean_spearman value: 37.895526338330484 - type: manhattan_pearson value: 38.116858432886445 - type: manhattan_spearman value: 37.88106872717212 - task: type: STS dataset: type: C-MTEB/QBQTC name: MTEB QBQTC config: default split: test revision: 790b0510dc52b1553e8c49f3d2afb48c0e5c48b7 metrics: - type: cos_sim_pearson value: 32.1385659529461 - type: cos_sim_spearman value: 33.68856769791845 - type: euclidean_pearson value: 32.15616490172132 - type: euclidean_spearman value: 33.68858640049767 - type: manhattan_pearson value: 32.14550781906868 - type: manhattan_spearman value: 33.65866728414211 - task: type: STS dataset: type: mteb/sts22-crosslingual-sts name: MTEB STS22 (zh) config: zh split: test revision: eea2b4fe26a775864c896887d910b76a8098ad3f metrics: - type: cos_sim_pearson value: 63.64041828778335 - type: cos_sim_spearman value: 64.3174530347361 - type: euclidean_pearson value: 63.479188950744536 - type: euclidean_spearman value: 64.3174530347361 - type: manhattan_pearson value: 64.39947888986349 - type: manhattan_spearman value: 64.85543965969501 - task: type: STS dataset: type: C-MTEB/STSB name: MTEB STSB config: default split: test revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0 metrics: - type: cos_sim_pearson value: 79.7474147570934 - type: cos_sim_spearman value: 81.18400577201275 - type: euclidean_pearson value: 80.94389968174902 - type: euclidean_spearman value: 81.1838334524369 - type: manhattan_pearson value: 80.88029725713798 - type: manhattan_spearman value: 81.12169479476515 - task: type: Reranking dataset: type: C-MTEB/T2Reranking name: MTEB T2Reranking config: default split: dev revision: 76631901a18387f85eaa53e5450019b87ad58ef9 metrics: - type: map value: 66.59735366130352 - type: mrr value: 76.42600563132046 - task: type: Retrieval dataset: type: C-MTEB/T2Retrieval name: MTEB T2Retrieval config: default split: dev revision: 8731a845f1bf500a4f111cf1070785c793d10e64 metrics: - type: map_at_1 value: 28.819 - type: map_at_10 value: 77.486 - type: map_at_100 value: 81.14 - type: map_at_1000 value: 81.21300000000001 - type: map_at_3 value: 55.432 - type: map_at_5 value: 67.486 - type: mrr_at_1 value: 91.364 - type: mrr_at_10 value: 94.059 - type: mrr_at_100 value: 94.118 - type: mrr_at_1000 value: 94.119 - type: mrr_at_3 value: 93.705 - type: mrr_at_5 value: 93.928 - type: ndcg_at_1 value: 91.364 - type: ndcg_at_10 value: 85.446 - type: ndcg_at_100 value: 89.10900000000001 - type: ndcg_at_1000 value: 89.75 - type: ndcg_at_3 value: 87.259 - type: ndcg_at_5 value: 85.658 - type: precision_at_1 value: 91.364 - type: precision_at_10 value: 41.525 - type: precision_at_100 value: 4.989 - type: precision_at_1000 value: 0.516 - type: precision_at_3 value: 75.44999999999999 - type: precision_at_5 value: 62.773999999999994 - type: recall_at_1 value: 28.819 - type: recall_at_10 value: 84.20400000000001 - type: recall_at_100 value: 95.773 - type: recall_at_1000 value: 98.79 - type: recall_at_3 value: 57.281000000000006 - type: recall_at_5 value: 71.06099999999999 - task: type: Classification dataset: type: C-MTEB/TNews-classification name: MTEB TNews config: default split: validation revision: 317f262bf1e6126357bbe89e875451e4b0938fe4 metrics: - type: accuracy value: 52.16399999999999 - type: f1 value: 50.49861014604211 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringP2P name: MTEB ThuNewsClusteringP2P config: default split: test revision: 5798586b105c0434e4f0fe5e767abe619442cf93 metrics: - type: v_measure value: 70.29645586740038 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringS2S name: MTEB ThuNewsClusteringS2S config: default split: test revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d metrics: - type: v_measure value: 62.99298994761416 - task: type: Retrieval dataset: type: C-MTEB/VideoRetrieval name: MTEB VideoRetrieval config: default split: dev revision: 58c2597a5943a2ba48f4668c3b90d796283c5639 metrics: - type: map_at_1 value: 60.099999999999994 - type: map_at_10 value: 69.404 - type: map_at_100 value: 69.85900000000001 - type: map_at_1000 value: 69.87299999999999 - type: map_at_3 value: 67.80000000000001 - type: map_at_5 value: 68.755 - type: mrr_at_1 value: 60.099999999999994 - type: mrr_at_10 value: 69.404 - type: mrr_at_100 value: 69.85900000000001 - type: mrr_at_1000 value: 69.87299999999999 - type: mrr_at_3 value: 67.80000000000001 - type: mrr_at_5 value: 68.755 - type: ndcg_at_1 value: 60.099999999999994 - type: ndcg_at_10 value: 73.624 - type: ndcg_at_100 value: 75.763 - type: ndcg_at_1000 value: 76.077 - type: ndcg_at_3 value: 70.316 - type: ndcg_at_5 value: 72.063 - type: precision_at_1 value: 60.099999999999994 - type: precision_at_10 value: 8.67 - type: precision_at_100 value: 0.9650000000000001 - type: precision_at_1000 value: 0.099 - type: precision_at_3 value: 25.867 - type: precision_at_5 value: 16.38 - type: recall_at_1 value: 60.099999999999994 - type: recall_at_10 value: 86.7 - type: recall_at_100 value: 96.5 - type: recall_at_1000 value: 98.9 - type: recall_at_3 value: 77.60000000000001 - type: recall_at_5 value: 81.89999999999999 - task: type: Classification dataset: type: C-MTEB/waimai-classification name: MTEB Waimai config: default split: test revision: 339287def212450dcaa9df8c22bf93e9980c7023 metrics: - type: accuracy value: 88.36 - type: ap value: 73.25144216855439 - type: f1 value: 86.75076261442027 ---