--- tags: - mteb model-index: - name: data1 results: - task: type: STS dataset: type: C-MTEB/AFQMC name: MTEB AFQMC config: default split: validation revision: b44c3b011063adb25877c13823db83bb193913c4 metrics: - type: cos_sim_pearson value: 53.66919706568301 - type: cos_sim_spearman value: 53.84074348656974 - type: euclidean_pearson value: 53.58226184439896 - type: euclidean_spearman value: 53.84074348656974 - type: manhattan_pearson value: 53.64565834381205 - type: manhattan_spearman value: 53.75526003581371 - task: type: STS dataset: type: C-MTEB/ATEC name: MTEB ATEC config: default split: test revision: 0f319b1142f28d00e055a6770f3f726ae9b7d865 metrics: - type: cos_sim_pearson value: 58.123744893539495 - type: cos_sim_spearman value: 54.44277675493291 - type: euclidean_pearson value: 61.20550691770944 - type: euclidean_spearman value: 54.44277225170509 - type: manhattan_pearson value: 60.57835645653918 - type: manhattan_spearman value: 54.46153709699013 - task: type: Classification dataset: type: mteb/amazon_reviews_multi name: MTEB AmazonReviewsClassification (zh) config: zh split: test revision: 1399c76144fd37290681b995c656ef9b2e06e26d metrics: - type: accuracy value: 29.746 - type: f1 value: 29.039321522193585 - task: type: STS dataset: type: C-MTEB/BQ name: MTEB BQ config: default split: test revision: e3dda5e115e487b39ec7e618c0c6a29137052a55 metrics: - type: cos_sim_pearson value: 70.7026320728244 - type: cos_sim_spearman value: 70.57218534128499 - type: euclidean_pearson value: 69.28488221289881 - type: euclidean_spearman value: 70.57218534192015 - type: manhattan_pearson value: 69.65344674392082 - type: manhattan_spearman value: 70.64136691477553 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringP2P name: MTEB CLSClusteringP2P config: default split: test revision: 4b6227591c6c1a73bc76b1055f3b7f3588e72476 metrics: - type: v_measure value: 38.87791994762536 - task: type: Clustering dataset: type: C-MTEB/CLSClusteringS2S name: MTEB CLSClusteringS2S config: default split: test revision: e458b3f5414b62b7f9f83499ac1f5497ae2e869f metrics: - type: v_measure value: 39.09103599244803 - task: type: Reranking dataset: type: C-MTEB/CMedQAv1-reranking name: MTEB CMedQAv1 config: default split: test revision: 8d7f1e942507dac42dc58017c1a001c3717da7df metrics: - type: map value: 80.40249793910444 - type: mrr value: 82.96805555555555 - task: type: Reranking dataset: type: C-MTEB/CMedQAv2-reranking name: MTEB CMedQAv2 config: default split: test revision: 23d186750531a14a0357ca22cd92d712fd512ea0 metrics: - type: map value: 80.39046823499085 - type: mrr value: 83.22674603174602 - task: type: Retrieval dataset: type: C-MTEB/CmedqaRetrieval name: MTEB CmedqaRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 15.715000000000002 - type: map_at_10 value: 24.651 - type: map_at_100 value: 26.478 - type: map_at_1000 value: 26.648 - type: map_at_3 value: 21.410999999999998 - type: map_at_5 value: 23.233 - type: mrr_at_1 value: 24.806 - type: mrr_at_10 value: 32.336 - type: mrr_at_100 value: 33.493 - type: mrr_at_1000 value: 33.568999999999996 - type: mrr_at_3 value: 29.807 - type: mrr_at_5 value: 31.294 - type: ndcg_at_1 value: 24.806 - type: ndcg_at_10 value: 30.341 - type: ndcg_at_100 value: 38.329 - type: ndcg_at_1000 value: 41.601 - type: ndcg_at_3 value: 25.655 - type: ndcg_at_5 value: 27.758 - type: precision_at_1 value: 24.806 - type: precision_at_10 value: 7.119000000000001 - type: precision_at_100 value: 1.3679999999999999 - type: precision_at_1000 value: 0.179 - type: precision_at_3 value: 14.787 - type: precision_at_5 value: 11.208 - type: recall_at_1 value: 15.715000000000002 - type: recall_at_10 value: 39.519999999999996 - type: recall_at_100 value: 73.307 - type: recall_at_1000 value: 95.611 - type: recall_at_3 value: 26.026 - type: recall_at_5 value: 32.027 - task: type: PairClassification dataset: type: C-MTEB/CMNLI name: MTEB Cmnli config: default split: validation revision: 41bc36f332156f7adc9e38f53777c959b2ae9766 metrics: - type: cos_sim_accuracy value: 66.89116055321708 - type: cos_sim_ap value: 75.66575745519994 - type: cos_sim_f1 value: 70.2448775612194 - type: cos_sim_precision value: 61.347765363128495 - type: cos_sim_recall value: 82.16039279869068 - type: dot_accuracy value: 66.89116055321708 - type: dot_ap value: 75.68262052264197 - type: dot_f1 value: 70.2448775612194 - type: dot_precision value: 61.347765363128495 - type: dot_recall value: 82.16039279869068 - type: euclidean_accuracy value: 66.89116055321708 - type: euclidean_ap value: 75.66576722188334 - type: euclidean_f1 value: 70.2448775612194 - type: euclidean_precision value: 61.347765363128495 - type: euclidean_recall value: 82.16039279869068 - type: manhattan_accuracy value: 67.03547805171377 - type: manhattan_ap value: 75.78816934864089 - type: manhattan_f1 value: 70.35407081416284 - type: manhattan_precision value: 61.4752665617899 - type: manhattan_recall value: 82.23053542202479 - type: max_accuracy value: 67.03547805171377 - type: max_ap value: 75.78816934864089 - type: max_f1 value: 70.35407081416284 - task: type: Retrieval dataset: type: C-MTEB/CovidRetrieval name: MTEB CovidRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 41.57 - type: map_at_10 value: 52.932 - type: map_at_100 value: 53.581999999999994 - type: map_at_1000 value: 53.61900000000001 - type: map_at_3 value: 50.066 - type: map_at_5 value: 51.735 - type: mrr_at_1 value: 41.623 - type: mrr_at_10 value: 52.964999999999996 - type: mrr_at_100 value: 53.6 - type: mrr_at_1000 value: 53.637 - type: mrr_at_3 value: 50.158 - type: mrr_at_5 value: 51.786 - type: ndcg_at_1 value: 41.623 - type: ndcg_at_10 value: 58.55200000000001 - type: ndcg_at_100 value: 61.824999999999996 - type: ndcg_at_1000 value: 62.854 - type: ndcg_at_3 value: 52.729000000000006 - type: ndcg_at_5 value: 55.696999999999996 - type: precision_at_1 value: 41.623 - type: precision_at_10 value: 7.692 - type: precision_at_100 value: 0.927 - type: precision_at_1000 value: 0.101 - type: precision_at_3 value: 20.162 - type: precision_at_5 value: 13.572000000000001 - type: recall_at_1 value: 41.57 - type: recall_at_10 value: 76.185 - type: recall_at_100 value: 91.728 - type: recall_at_1000 value: 99.895 - type: recall_at_3 value: 60.27400000000001 - type: recall_at_5 value: 67.46600000000001 - task: type: Retrieval dataset: type: C-MTEB/DuRetrieval name: MTEB DuRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 21.071 - type: map_at_10 value: 65.093 - type: map_at_100 value: 69.097 - type: map_at_1000 value: 69.172 - type: map_at_3 value: 44.568000000000005 - type: map_at_5 value: 56.016999999999996 - type: mrr_at_1 value: 76.35 - type: mrr_at_10 value: 83.721 - type: mrr_at_100 value: 83.899 - type: mrr_at_1000 value: 83.904 - type: mrr_at_3 value: 82.958 - type: mrr_at_5 value: 83.488 - type: ndcg_at_1 value: 76.35 - type: ndcg_at_10 value: 75.05199999999999 - type: ndcg_at_100 value: 80.596 - type: ndcg_at_1000 value: 81.394 - type: ndcg_at_3 value: 73.298 - type: ndcg_at_5 value: 72.149 - type: precision_at_1 value: 76.35 - type: precision_at_10 value: 36.96 - type: precision_at_100 value: 4.688 - type: precision_at_1000 value: 0.48700000000000004 - type: precision_at_3 value: 66.2 - type: precision_at_5 value: 55.81 - type: recall_at_1 value: 21.071 - type: recall_at_10 value: 77.459 - type: recall_at_100 value: 94.425 - type: recall_at_1000 value: 98.631 - type: recall_at_3 value: 48.335 - type: recall_at_5 value: 63.227999999999994 - task: type: Retrieval dataset: type: C-MTEB/EcomRetrieval name: MTEB EcomRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 36.3 - type: map_at_10 value: 46.888999999999996 - type: map_at_100 value: 47.789 - type: map_at_1000 value: 47.827999999999996 - type: map_at_3 value: 43.85 - type: map_at_5 value: 45.58 - type: mrr_at_1 value: 36.3 - type: mrr_at_10 value: 46.888999999999996 - type: mrr_at_100 value: 47.789 - type: mrr_at_1000 value: 47.827999999999996 - type: mrr_at_3 value: 43.85 - type: mrr_at_5 value: 45.58 - type: ndcg_at_1 value: 36.3 - type: ndcg_at_10 value: 52.539 - type: ndcg_at_100 value: 56.882 - type: ndcg_at_1000 value: 57.841 - type: ndcg_at_3 value: 46.303 - type: ndcg_at_5 value: 49.406 - type: precision_at_1 value: 36.3 - type: precision_at_10 value: 7.049999999999999 - type: precision_at_100 value: 0.907 - type: precision_at_1000 value: 0.098 - type: precision_at_3 value: 17.8 - type: precision_at_5 value: 12.18 - type: recall_at_1 value: 36.3 - type: recall_at_10 value: 70.5 - type: recall_at_100 value: 90.7 - type: recall_at_1000 value: 98.1 - type: recall_at_3 value: 53.400000000000006 - type: recall_at_5 value: 60.9 - task: type: Classification dataset: type: C-MTEB/IFlyTek-classification name: MTEB IFlyTek config: default split: validation revision: 421605374b29664c5fc098418fe20ada9bd55f8a metrics: - type: accuracy value: 50.927279722970376 - type: f1 value: 39.57514582425314 - task: type: Classification dataset: type: C-MTEB/JDReview-classification name: MTEB JDReview config: default split: test revision: b7c64bd89eb87f8ded463478346f76731f07bf8b metrics: - type: accuracy value: 84.93433395872421 - type: ap value: 50.35046267230439 - type: f1 value: 78.76452515604298 - task: type: STS dataset: type: C-MTEB/LCQMC name: MTEB LCQMC config: default split: test revision: 17f9b096f80380fce5ed12a9be8be7784b337daf metrics: - type: cos_sim_pearson value: 67.40319768112933 - type: cos_sim_spearman value: 74.9867527749418 - type: euclidean_pearson value: 74.08762625643878 - type: euclidean_spearman value: 74.98675720634276 - type: manhattan_pearson value: 73.86303861791671 - type: manhattan_spearman value: 75.0594224188492 - task: type: Reranking dataset: type: C-MTEB/Mmarco-reranking name: MTEB MMarcoReranking config: default split: dev revision: 8e0c766dbe9e16e1d221116a3f36795fbade07f6 metrics: - type: map value: 18.860945903258536 - type: mrr value: 17.686507936507937 - task: type: Retrieval dataset: type: C-MTEB/MMarcoRetrieval name: MTEB MMarcoRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 49.16 - type: map_at_10 value: 57.992 - type: map_at_100 value: 58.638 - type: map_at_1000 value: 58.67 - type: map_at_3 value: 55.71 - type: map_at_5 value: 57.04900000000001 - type: mrr_at_1 value: 50.989 - type: mrr_at_10 value: 58.814 - type: mrr_at_100 value: 59.401 - type: mrr_at_1000 value: 59.431 - type: mrr_at_3 value: 56.726 - type: mrr_at_5 value: 57.955 - type: ndcg_at_1 value: 50.989 - type: ndcg_at_10 value: 62.259 - type: ndcg_at_100 value: 65.347 - type: ndcg_at_1000 value: 66.231 - type: ndcg_at_3 value: 57.78 - type: ndcg_at_5 value: 60.09100000000001 - type: precision_at_1 value: 50.989 - type: precision_at_10 value: 7.9479999999999995 - type: precision_at_100 value: 0.951 - type: precision_at_1000 value: 0.10200000000000001 - type: precision_at_3 value: 22.087 - type: precision_at_5 value: 14.479000000000001 - type: recall_at_1 value: 49.16 - type: recall_at_10 value: 74.792 - type: recall_at_100 value: 89.132 - type: recall_at_1000 value: 96.13199999999999 - type: recall_at_3 value: 62.783 - type: recall_at_5 value: 68.26100000000001 - task: type: Classification dataset: type: mteb/amazon_massive_intent name: MTEB MassiveIntentClassification (zh-CN) config: zh-CN split: test revision: 31efe3c427b0bae9c22cbb560b8f15491cc6bed7 metrics: - type: accuracy value: 67.45796906523202 - type: f1 value: 65.97280169222601 - task: type: Classification dataset: type: mteb/amazon_massive_scenario name: MTEB MassiveScenarioClassification (zh-CN) config: zh-CN split: test revision: 7d571f92784cd94a019292a1f45445077d0ef634 metrics: - type: accuracy value: 71.59717552118359 - type: f1 value: 72.46681610207507 - task: type: Retrieval dataset: type: C-MTEB/MedicalRetrieval name: MTEB MedicalRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 40.5 - type: map_at_10 value: 46.892 - type: map_at_100 value: 47.579 - type: map_at_1000 value: 47.648 - type: map_at_3 value: 45.367000000000004 - type: map_at_5 value: 46.182 - type: mrr_at_1 value: 40.6 - type: mrr_at_10 value: 46.942 - type: mrr_at_100 value: 47.629 - type: mrr_at_1000 value: 47.698 - type: mrr_at_3 value: 45.417 - type: mrr_at_5 value: 46.232 - type: ndcg_at_1 value: 40.5 - type: ndcg_at_10 value: 50.078 - type: ndcg_at_100 value: 53.635999999999996 - type: ndcg_at_1000 value: 55.696999999999996 - type: ndcg_at_3 value: 46.847 - type: ndcg_at_5 value: 48.323 - type: precision_at_1 value: 40.5 - type: precision_at_10 value: 6.02 - type: precision_at_100 value: 0.773 - type: precision_at_1000 value: 0.094 - type: precision_at_3 value: 17.033 - type: precision_at_5 value: 10.94 - type: recall_at_1 value: 40.5 - type: recall_at_10 value: 60.199999999999996 - type: recall_at_100 value: 77.3 - type: recall_at_1000 value: 94.0 - type: recall_at_3 value: 51.1 - type: recall_at_5 value: 54.7 - task: type: Retrieval dataset: type: Shitao/MLDR name: MTEB MultiLongDocRetrieval (zh) config: zh split: test revision: None metrics: - type: map_at_1 value: 7.000000000000001 - type: map_at_10 value: 10.020999999999999 - type: map_at_100 value: 10.511 - type: map_at_1000 value: 10.595 - type: map_at_3 value: 9.042 - type: map_at_5 value: 9.654 - type: mrr_at_1 value: 6.875000000000001 - type: mrr_at_10 value: 9.958 - type: mrr_at_100 value: 10.449 - type: mrr_at_1000 value: 10.532 - type: mrr_at_3 value: 8.979 - type: mrr_at_5 value: 9.592 - type: ndcg_at_1 value: 7.000000000000001 - type: ndcg_at_10 value: 11.651 - type: ndcg_at_100 value: 14.580000000000002 - type: ndcg_at_1000 value: 17.183 - type: ndcg_at_3 value: 9.646 - type: ndcg_at_5 value: 10.738 - type: precision_at_1 value: 7.000000000000001 - type: precision_at_10 value: 1.687 - type: precision_at_100 value: 0.319 - type: precision_at_1000 value: 0.053 - type: precision_at_3 value: 3.7920000000000003 - type: precision_at_5 value: 2.8000000000000003 - type: recall_at_1 value: 7.000000000000001 - type: recall_at_10 value: 16.875 - type: recall_at_100 value: 31.874999999999996 - type: recall_at_1000 value: 53.25 - type: recall_at_3 value: 11.375 - type: recall_at_5 value: 14.000000000000002 - task: type: Classification dataset: type: C-MTEB/MultilingualSentiment-classification name: MTEB MultilingualSentiment config: default split: validation revision: 46958b007a63fdbf239b7672c25d0bea67b5ea1a metrics: - type: accuracy value: 55.90333333333333 - type: f1 value: 55.291185234519546 - task: type: PairClassification dataset: type: C-MTEB/OCNLI name: MTEB Ocnli config: default split: validation revision: 66e76a618a34d6d565d5538088562851e6daa7ec metrics: - type: cos_sim_accuracy value: 59.01461829994585 - type: cos_sim_ap value: 61.84829541140869 - type: cos_sim_f1 value: 67.94150731158605 - type: cos_sim_precision value: 52.674418604651166 - type: cos_sim_recall value: 95.67053854276664 - type: dot_accuracy value: 59.01461829994585 - type: dot_ap value: 61.84829541140869 - type: dot_f1 value: 67.94150731158605 - type: dot_precision value: 52.674418604651166 - type: dot_recall value: 95.67053854276664 - type: euclidean_accuracy value: 59.01461829994585 - type: euclidean_ap value: 61.84829541140869 - type: euclidean_f1 value: 67.94150731158605 - type: euclidean_precision value: 52.674418604651166 - type: euclidean_recall value: 95.67053854276664 - type: manhattan_accuracy value: 59.06876015159719 - type: manhattan_ap value: 61.91217952354554 - type: manhattan_f1 value: 67.89059572873735 - type: manhattan_precision value: 52.613240418118465 - type: manhattan_recall value: 95.67053854276664 - type: max_accuracy value: 59.06876015159719 - type: max_ap value: 61.91217952354554 - type: max_f1 value: 67.94150731158605 - task: type: Classification dataset: type: C-MTEB/OnlineShopping-classification name: MTEB OnlineShopping config: default split: test revision: e610f2ebd179a8fda30ae534c3878750a96db120 metrics: - type: accuracy value: 82.53 - type: ap value: 77.67591637020448 - type: f1 value: 82.39976599130478 - task: type: STS dataset: type: C-MTEB/PAWSX name: MTEB PAWSX config: default split: test revision: 9c6a90e430ac22b5779fb019a23e820b11a8b5e1 metrics: - type: cos_sim_pearson value: 55.76388035743312 - type: cos_sim_spearman value: 58.34768166139753 - type: euclidean_pearson value: 57.971763429924074 - type: euclidean_spearman value: 58.34750745303424 - type: manhattan_pearson value: 58.044053497280245 - type: manhattan_spearman value: 58.61627719613188 - task: type: PairClassification dataset: type: paws-x name: MTEB PawsX (zh) config: zh split: test revision: 8a04d940a42cd40658986fdd8e3da561533a3646 metrics: - type: cos_sim_accuracy value: 75.75 - type: cos_sim_ap value: 78.80617392926526 - type: cos_sim_f1 value: 75.92417061611374 - type: cos_sim_precision value: 65.87171052631578 - type: cos_sim_recall value: 89.59731543624162 - type: dot_accuracy value: 75.75 - type: dot_ap value: 78.83768586994135 - type: dot_f1 value: 75.92417061611374 - type: dot_precision value: 65.87171052631578 - type: dot_recall value: 89.59731543624162 - type: euclidean_accuracy value: 75.75 - type: euclidean_ap value: 78.80617392926526 - type: euclidean_f1 value: 75.92417061611374 - type: euclidean_precision value: 65.87171052631578 - type: euclidean_recall value: 89.59731543624162 - type: manhattan_accuracy value: 75.75 - type: manhattan_ap value: 78.98640478955386 - type: manhattan_f1 value: 75.92954990215264 - type: manhattan_precision value: 67.47826086956522 - type: manhattan_recall value: 86.80089485458613 - type: max_accuracy value: 75.75 - type: max_ap value: 78.98640478955386 - type: max_f1 value: 75.92954990215264 - task: type: STS dataset: type: C-MTEB/QBQTC name: MTEB QBQTC config: default split: test revision: 790b0510dc52b1553e8c49f3d2afb48c0e5c48b7 metrics: - type: cos_sim_pearson value: 74.40348414238575 - type: cos_sim_spearman value: 71.452270332177 - type: euclidean_pearson value: 72.62509231589097 - type: euclidean_spearman value: 71.45228258458943 - type: manhattan_pearson value: 73.03846856200839 - type: manhattan_spearman value: 71.43673225319574 - task: type: STS dataset: type: mteb/sts22-crosslingual-sts name: MTEB STS22 (zh) config: zh split: test revision: eea2b4fe26a775864c896887d910b76a8098ad3f metrics: - type: cos_sim_pearson value: 75.38335474357001 - type: cos_sim_spearman value: 74.92262892309807 - type: euclidean_pearson value: 73.93451693251345 - type: euclidean_spearman value: 74.92262892309807 - type: manhattan_pearson value: 74.55911294300788 - type: manhattan_spearman value: 74.89436791272614 - task: type: STS dataset: type: C-MTEB/STSB name: MTEB STSB config: default split: test revision: 0cde68302b3541bb8b3c340dc0644b0b745b3dc0 metrics: - type: cos_sim_pearson value: 83.01687361650126 - type: cos_sim_spearman value: 82.74413230806265 - type: euclidean_pearson value: 81.50177295189083 - type: euclidean_spearman value: 82.74413230806265 - type: manhattan_pearson value: 81.90798387028589 - type: manhattan_spearman value: 82.65064251275778 - task: type: Reranking dataset: type: C-MTEB/T2Reranking name: MTEB T2Reranking config: default split: dev revision: 76631901a18387f85eaa53e5450019b87ad58ef9 metrics: - type: map value: 66.25459669294304 - type: mrr value: 76.76845224661744 - task: type: Retrieval dataset: type: C-MTEB/T2Retrieval name: MTEB T2Retrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 22.515 - type: map_at_10 value: 63.63999999999999 - type: map_at_100 value: 67.67 - type: map_at_1000 value: 67.792 - type: map_at_3 value: 44.239 - type: map_at_5 value: 54.54599999999999 - type: mrr_at_1 value: 79.752 - type: mrr_at_10 value: 83.525 - type: mrr_at_100 value: 83.753 - type: mrr_at_1000 value: 83.763 - type: mrr_at_3 value: 82.65599999999999 - type: mrr_at_5 value: 83.192 - type: ndcg_at_1 value: 79.752 - type: ndcg_at_10 value: 72.699 - type: ndcg_at_100 value: 78.145 - type: ndcg_at_1000 value: 79.481 - type: ndcg_at_3 value: 74.401 - type: ndcg_at_5 value: 72.684 - type: precision_at_1 value: 79.752 - type: precision_at_10 value: 37.163000000000004 - type: precision_at_100 value: 4.769 - type: precision_at_1000 value: 0.508 - type: precision_at_3 value: 65.67399999999999 - type: precision_at_5 value: 55.105000000000004 - type: recall_at_1 value: 22.515 - type: recall_at_10 value: 71.816 - type: recall_at_100 value: 89.442 - type: recall_at_1000 value: 96.344 - type: recall_at_3 value: 46.208 - type: recall_at_5 value: 58.695 - task: type: Classification dataset: type: C-MTEB/TNews-classification name: MTEB TNews config: default split: validation revision: 317f262bf1e6126357bbe89e875451e4b0938fe4 metrics: - type: accuracy value: 55.077999999999996 - type: f1 value: 53.2447237349446 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringP2P name: MTEB ThuNewsClusteringP2P config: default split: test revision: 5798586b105c0434e4f0fe5e767abe619442cf93 metrics: - type: v_measure value: 59.50582115422618 - task: type: Clustering dataset: type: C-MTEB/ThuNewsClusteringS2S name: MTEB ThuNewsClusteringS2S config: default split: test revision: 8a8b2caeda43f39e13c4bc5bea0f8a667896e10d metrics: - type: v_measure value: 54.71907850412647 - task: type: Retrieval dataset: type: C-MTEB/VideoRetrieval name: MTEB VideoRetrieval config: default split: dev revision: None metrics: - type: map_at_1 value: 49.4 - type: map_at_10 value: 59.245999999999995 - type: map_at_100 value: 59.811 - type: map_at_1000 value: 59.836 - type: map_at_3 value: 56.733 - type: map_at_5 value: 58.348 - type: mrr_at_1 value: 49.4 - type: mrr_at_10 value: 59.245999999999995 - type: mrr_at_100 value: 59.811 - type: mrr_at_1000 value: 59.836 - type: mrr_at_3 value: 56.733 - type: mrr_at_5 value: 58.348 - type: ndcg_at_1 value: 49.4 - type: ndcg_at_10 value: 64.08 - type: ndcg_at_100 value: 67.027 - type: ndcg_at_1000 value: 67.697 - type: ndcg_at_3 value: 58.995 - type: ndcg_at_5 value: 61.891 - type: precision_at_1 value: 49.4 - type: precision_at_10 value: 7.93 - type: precision_at_100 value: 0.935 - type: precision_at_1000 value: 0.099 - type: precision_at_3 value: 21.833 - type: precision_at_5 value: 14.499999999999998 - type: recall_at_1 value: 49.4 - type: recall_at_10 value: 79.3 - type: recall_at_100 value: 93.5 - type: recall_at_1000 value: 98.8 - type: recall_at_3 value: 65.5 - type: recall_at_5 value: 72.5 - task: type: Classification dataset: type: C-MTEB/waimai-classification name: MTEB Waimai config: default split: test revision: 339287def212450dcaa9df8c22bf93e9980c7023 metrics: - type: accuracy value: 81.16 - type: ap value: 60.864524843400616 - type: f1 value: 79.41246877404483 --- ZNV Embedding utilizes a 6B LLM (Large Language Model) for embedding, achieving excellent embedding results. In a single inference, we used two prompts to extract two different embeddings for a sentence, and then concatenated them. Model usage method: 1. Define ZNVEmbeddingModel ```python import os from transformers import ( LlamaForCausalLM, LlamaTokenizer, AutoConfig, ) import torch import torch.nn.functional as F import numpy as np class ZNVEmbeddingModel(torch.nn.Module): def __init__(self, model_name_or_path): super(ZNVEmbeddingModel, self).__init__() self.prompt_prefix = "阅读下文，然后答题\n" self.prompt_suffixes = ["\n1.一个字总结上文的意思是:", "\n2.上文深层次的意思是:"] self.hidden_size = 4096 self.model_name_or_path = model_name_or_path self.linear_suffixes = torch.nn.ModuleList( [torch.nn.Linear(self.hidden_size, self.hidden_size//len(self.prompt_suffixes)) for _ in range(len(self.prompt_suffixes))]) self.tokenizer, self.llama = self.load_llama() self.tanh = torch.nn.Tanh() self.suffixes_ids = [] self.suffixes_ids_len = [] self.suffixes_len = 0 for suffix in self.prompt_suffixes: ids = self.tokenizer(suffix, return_tensors="pt")["input_ids"].tolist()[0] self.suffixes_ids += ids self.suffixes_ids_len.append(len(ids)) self.suffixes_len += len(ids) self.suffixes_ones = torch.ones(self.suffixes_len) self.suffixes_ids = torch.tensor(self.suffixes_ids) linear_file = os.path.join(model_name_or_path, "linears") load_layers = torch.load(linear_file) model_state = self.state_dict() model_state.update(load_layers) self.load_state_dict(model_state, strict=False) def load_llama(self): llm_path = os.path.join(self.model_name_or_path) config = AutoConfig.from_pretrained(llm_path) tokenizer = LlamaTokenizer.from_pretrained(self.model_name_or_path) tokenizer.padding_side = "left" model = LlamaForCausalLM.from_pretrained( llm_path, config=config, low_cpu_mem_usage=True ) model.config.use_cache = False return tokenizer, model def forward(self, sentences): prompts_embeddings = [] sentences = [self.prompt_prefix + s for s in sentences] inputs = self.tokenizer(sentences, max_length=256, padding=True, truncation=True, return_tensors='pt') attention_mask = inputs["attention_mask"] input_ids = inputs["input_ids"] batch_size = len(sentences) suffixes_ones = self.suffixes_ones.unsqueeze(0) suffixes_ones = suffixes_ones.repeat(batch_size, 1) device = next(self.parameters()).device attention_mask = torch.cat([attention_mask, suffixes_ones], dim=-1).to(device) suffixes_ids = self.suffixes_ids.unsqueeze(0) suffixes_ids = suffixes_ids.repeat(batch_size, 1) input_ids = torch.cat([input_ids, suffixes_ids], dim=-1).to(device) last_hidden_state = self.llama.base_model.base_model(attention_mask=attention_mask, input_ids=input_ids).last_hidden_state index = -1 for i in range(len(self.suffixes_ids_len)): embedding = last_hidden_state[:, index, :] embedding = self.linear_suffixes[i](embedding) prompts_embeddings.append(embedding) index -= self.suffixes_ids_len[-i-1] output_embedding = torch.cat(prompts_embeddings, dim=-1) output_embedding = self.tanh(output_embedding) output_embedding = F.normalize(output_embedding, p=2, dim=1) return output_embedding def encode(self, sentences, batch_size=10, **kwargs): size = len(sentences) embeddings = None handled = 0 while handled < size: tokens = sentences[handled:handled + batch_size] output_embeddings = self.forward(tokens) result = output_embeddings.cpu().numpy() handled += result.shape[0] if embeddings is not None: embeddings = np.concatenate((embeddings, result), axis=0) else: embeddings = result return embeddings ``` 2. Use ZNVEmbeddingModel for Embedding. ```python znv_model = ZNVEmbeddingModel("your_model_path") znv_model.eval() with torch.no_grad(): output = znv_model(["请问你的电话号码是多少？","可以告诉我你的手机号吗？"]) cos_sim = F.cosine_similarity(output[0],output[1],dim=0) print(cos_sim) ```