Update README.md
Browse files
README.md
CHANGED
@@ -1207,7 +1207,7 @@ model-index:
|
|
1207 |
- type: mrr_at_5
|
1208 |
value: 74.012
|
1209 |
- type: ndcg_at_1
|
1210 |
-
value: 54
|
1211 |
- type: ndcg_at_10
|
1212 |
value: 42.014
|
1213 |
- type: ndcg_at_100
|
@@ -2155,7 +2155,7 @@ model-index:
|
|
2155 |
- type: map_at_5
|
2156 |
value: 67.06299999999999
|
2157 |
- type: mrr_at_1
|
2158 |
-
value: 61
|
2159 |
- type: mrr_at_10
|
2160 |
value: 69.45400000000001
|
2161 |
- type: mrr_at_100
|
@@ -2163,11 +2163,11 @@ model-index:
|
|
2163 |
- type: mrr_at_1000
|
2164 |
value: 69.807
|
2165 |
- type: mrr_at_3
|
2166 |
-
value: 67
|
2167 |
- type: mrr_at_5
|
2168 |
value: 68.43299999999999
|
2169 |
- type: ndcg_at_1
|
2170 |
-
value: 61
|
2171 |
- type: ndcg_at_10
|
2172 |
value: 73.258
|
2173 |
- type: ndcg_at_100
|
@@ -2179,7 +2179,7 @@ model-index:
|
|
2179 |
- type: ndcg_at_5
|
2180 |
value: 70.53399999999999
|
2181 |
- type: precision_at_1
|
2182 |
-
value: 61
|
2183 |
- type: precision_at_10
|
2184 |
value: 9.8
|
2185 |
- type: precision_at_100
|
@@ -2187,7 +2187,7 @@ model-index:
|
|
2187 |
- type: precision_at_1000
|
2188 |
value: 0.11299999999999999
|
2189 |
- type: precision_at_3
|
2190 |
-
value: 27
|
2191 |
- type: precision_at_5
|
2192 |
value: 17.666999999999998
|
2193 |
- type: recall_at_1
|
@@ -2331,7 +2331,7 @@ model-index:
|
|
2331 |
- type: map_at_5
|
2332 |
value: 1.0290000000000001
|
2333 |
- type: mrr_at_1
|
2334 |
-
value: 88
|
2335 |
- type: mrr_at_10
|
2336 |
value: 93.5
|
2337 |
- type: mrr_at_100
|
@@ -2339,11 +2339,11 @@ model-index:
|
|
2339 |
- type: mrr_at_1000
|
2340 |
value: 93.5
|
2341 |
- type: mrr_at_3
|
2342 |
-
value: 93
|
2343 |
- type: mrr_at_5
|
2344 |
value: 93.5
|
2345 |
- type: ndcg_at_1
|
2346 |
-
value: 84
|
2347 |
- type: ndcg_at_10
|
2348 |
value: 76.447
|
2349 |
- type: ndcg_at_100
|
@@ -2355,7 +2355,7 @@ model-index:
|
|
2355 |
- type: ndcg_at_5
|
2356 |
value: 79.174
|
2357 |
- type: precision_at_1
|
2358 |
-
value: 88
|
2359 |
- type: precision_at_10
|
2360 |
value: 80.60000000000001
|
2361 |
- type: precision_at_100
|
@@ -2363,7 +2363,7 @@ model-index:
|
|
2363 |
- type: precision_at_1000
|
2364 |
value: 21.227999999999998
|
2365 |
- type: precision_at_3
|
2366 |
-
value: 82
|
2367 |
- type: precision_at_5
|
2368 |
value: 83.6
|
2369 |
- type: recall_at_1
|
@@ -2596,4 +2596,242 @@ model-index:
|
|
2596 |
value: 85.20370297495491
|
2597 |
- type: max_f1
|
2598 |
value: 77.01372369624886
|
2599 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1207 |
- type: mrr_at_5
|
1208 |
value: 74.012
|
1209 |
- type: ndcg_at_1
|
1210 |
+
value: 54
|
1211 |
- type: ndcg_at_10
|
1212 |
value: 42.014
|
1213 |
- type: ndcg_at_100
|
|
|
2155 |
- type: map_at_5
|
2156 |
value: 67.06299999999999
|
2157 |
- type: mrr_at_1
|
2158 |
+
value: 61
|
2159 |
- type: mrr_at_10
|
2160 |
value: 69.45400000000001
|
2161 |
- type: mrr_at_100
|
|
|
2163 |
- type: mrr_at_1000
|
2164 |
value: 69.807
|
2165 |
- type: mrr_at_3
|
2166 |
+
value: 67
|
2167 |
- type: mrr_at_5
|
2168 |
value: 68.43299999999999
|
2169 |
- type: ndcg_at_1
|
2170 |
+
value: 61
|
2171 |
- type: ndcg_at_10
|
2172 |
value: 73.258
|
2173 |
- type: ndcg_at_100
|
|
|
2179 |
- type: ndcg_at_5
|
2180 |
value: 70.53399999999999
|
2181 |
- type: precision_at_1
|
2182 |
+
value: 61
|
2183 |
- type: precision_at_10
|
2184 |
value: 9.8
|
2185 |
- type: precision_at_100
|
|
|
2187 |
- type: precision_at_1000
|
2188 |
value: 0.11299999999999999
|
2189 |
- type: precision_at_3
|
2190 |
+
value: 27
|
2191 |
- type: precision_at_5
|
2192 |
value: 17.666999999999998
|
2193 |
- type: recall_at_1
|
|
|
2331 |
- type: map_at_5
|
2332 |
value: 1.0290000000000001
|
2333 |
- type: mrr_at_1
|
2334 |
+
value: 88
|
2335 |
- type: mrr_at_10
|
2336 |
value: 93.5
|
2337 |
- type: mrr_at_100
|
|
|
2339 |
- type: mrr_at_1000
|
2340 |
value: 93.5
|
2341 |
- type: mrr_at_3
|
2342 |
+
value: 93
|
2343 |
- type: mrr_at_5
|
2344 |
value: 93.5
|
2345 |
- type: ndcg_at_1
|
2346 |
+
value: 84
|
2347 |
- type: ndcg_at_10
|
2348 |
value: 76.447
|
2349 |
- type: ndcg_at_100
|
|
|
2355 |
- type: ndcg_at_5
|
2356 |
value: 79.174
|
2357 |
- type: precision_at_1
|
2358 |
+
value: 88
|
2359 |
- type: precision_at_10
|
2360 |
value: 80.60000000000001
|
2361 |
- type: precision_at_100
|
|
|
2363 |
- type: precision_at_1000
|
2364 |
value: 21.227999999999998
|
2365 |
- type: precision_at_3
|
2366 |
+
value: 82
|
2367 |
- type: precision_at_5
|
2368 |
value: 83.6
|
2369 |
- type: recall_at_1
|
|
|
2596 |
value: 85.20370297495491
|
2597 |
- type: max_f1
|
2598 |
value: 77.01372369624886
|
2599 |
+
license: mit
|
2600 |
+
language:
|
2601 |
+
- en
|
2602 |
+
pipeline_tag: sentence-similarity
|
2603 |
+
---
|
2604 |
+
|
2605 |
+
<h1 align="center">FlagEmbedding</h1>
|
2606 |
+
|
2607 |
+
|
2608 |
+
<h4 align="center">
|
2609 |
+
<p>
|
2610 |
+
<a href=#model-list>Model List</a> |
|
2611 |
+
<a href=#usage>Usage</a> |
|
2612 |
+
<a href="#evaluation">Evaluation</a> |
|
2613 |
+
<a href="#train">Train</a> |
|
2614 |
+
<a href="#contact">Contact</a> |
|
2615 |
+
<a href="#license">License</a>
|
2616 |
+
<p>
|
2617 |
+
</h4>
|
2618 |
+
|
2619 |
+
More details please refer to our github: [FlagEmbedding](https://github.com/FlagOpen/FlagEmbedding).
|
2620 |
+
|
2621 |
+
[English](README.md) | [中文](README_zh.md)
|
2622 |
+
|
2623 |
+
FlagEmbedding can map any text to a low-dimensional dense vector which can be used for tasks like retrieval, classification, clustering, or semantic search.
|
2624 |
+
And it also can be used in vector database for LLMs.
|
2625 |
+
|
2626 |
+
************* 🌟**Updates**🌟 *************
|
2627 |
+
- 08/05/2023: Release base-scale and small-scale models, **best performance among the models of the same size 🤗**
|
2628 |
+
- 08/02/2023: Release `bge-large-*`(short for BAAI General Embedding) Models, **rank 1st on MTEB and C-MTEB benchmark!**
|
2629 |
+
- 08/01/2023: We release the Chinese Massive Text Embedding Benchmark (**C-MTEB**), consisting of 31 test dataset.
|
2630 |
+
|
2631 |
+
|
2632 |
+
## Model List
|
2633 |
+
|
2634 |
+
`bge` is short for `BAAI general embedding`.
|
2635 |
+
|
2636 |
+
| Model | Language | Description | query instruction for retrieval |
|
2637 |
+
|:-------------------------------|:--------:| :--------:| :--------:|
|
2638 |
+
| [BAAI/bge-large-en](https://huggingface.co/BAAI/bge-large-en) | English | **rank 1st** in [MTEB](https://huggingface.co/spaces/mteb/leaderboard) leaderboard | `Represent this sentence for searching relevant passages: ` |
|
2639 |
+
| [BAAI/bge-base-en](https://huggingface.co/BAAI/bge-base-en) | English | **rank 2nd** in [MTEB](https://huggingface.co/spaces/mteb/leaderboard) leaderboard | `Represent this sentence for searching relevant passages: ` |
|
2640 |
+
| [BAAI/bge-small-en](https://huggingface.co/BAAI/bge-small-en) | English | a small-scale model but with competitive performance | `Represent this sentence for searching relevant passages: ` |
|
2641 |
+
| [BAAI/bge-large-zh](https://huggingface.co/BAAI/bge-large-zh) | Chinese | **rank 1st** in [C-MTEB](https://github.com/FlagOpen/FlagEmbedding/tree/master/benchmark) benchmark | `为这个句子生成表示以用于检索相关文章:` |
|
2642 |
+
| [BAAI/bge-large-zh-noinstruct](https://huggingface.co/BAAI/bge-large-zh-noinstruct) | Chinese | This model is trained without instruction, and **rank 2nd** in [C-MTEB](https://github.com/FlagOpen/FlagEmbedding/tree/master/benchmark) benchmark | |
|
2643 |
+
| [BAAI/bge-base-zh](https://huggingface.co/BAAI/bge-base-zh) | Chinese | a base-scale model but with competitive performance | `为这个句子���成表示以用于检索相关文章:` |
|
2644 |
+
| [BAAI/bge-small-zh](https://huggingface.co/BAAI/bge-small-zh) | Chinese | a small-scale model but with competitive performance | `为这个句子生成表示以用于检索相关文章:` |
|
2645 |
+
|
2646 |
+
|
2647 |
+
|
2648 |
+
## Usage
|
2649 |
+
|
2650 |
+
* **Using FlagEmbedding**
|
2651 |
+
```
|
2652 |
+
pip install flag_embedding
|
2653 |
+
```
|
2654 |
+
```python
|
2655 |
+
from flag_embedding import FlagModel
|
2656 |
+
sentences = ["样例数据-1", "样例数据-2"]
|
2657 |
+
model = FlagModel('BAAI/bge-large-zh', query_instruction_for_retrieval="为这个句子生成表示以用于检索相关文章:")
|
2658 |
+
embeddings = model.encode(sentences)
|
2659 |
+
print(embeddings)
|
2660 |
+
|
2661 |
+
# for retrieval task, please use encode_queries() which will automatically add the instruction to each query
|
2662 |
+
# corpus in retrieval task can still use encode() or encode_corpus()
|
2663 |
+
queries = ['query_1', 'query_2']
|
2664 |
+
passages = ["样例段落-1", "样例段落-2"]
|
2665 |
+
q_embeddings = model.encode_queries(queries)
|
2666 |
+
p_embeddings = model.encode(passages)
|
2667 |
+
scores = q_embeddings @ p_embeddings.T
|
2668 |
+
```
|
2669 |
+
The value of argument `query_instruction_for_retrieval` see [Model List](https://github.com/FlagOpen/FlagEmbedding/tree/master#model-list).
|
2670 |
+
|
2671 |
+
FlagModel will use all available GPUs when encoding, please set `os.environ["CUDA_VISIBLE_DEVICES"]` to choose GPU.
|
2672 |
+
|
2673 |
+
|
2674 |
+
* **Using Sentence-Transformers**
|
2675 |
+
|
2676 |
+
Using this model also is easy when you have [sentence-transformers](https://www.SBERT.net) installed:
|
2677 |
+
|
2678 |
+
```
|
2679 |
+
pip install -U sentence-transformers
|
2680 |
+
```
|
2681 |
+
```python
|
2682 |
+
from sentence_transformers import SentenceTransformer
|
2683 |
+
sentences = ["样例数据-1", "样例数据-2"]
|
2684 |
+
model = SentenceTransformer('BAAI/bge-large-zh')
|
2685 |
+
embeddings = model.encode(sentences, normalize_embeddings=True)
|
2686 |
+
print(embeddings)
|
2687 |
+
```
|
2688 |
+
For retrieval task,
|
2689 |
+
each query should start with a instruction (instructions see [Model List](https://github.com/FlagOpen/FlagEmbedding/tree/master#model-list)).
|
2690 |
+
```python
|
2691 |
+
from sentence_transformers import SentenceTransformer
|
2692 |
+
queries = ["手机开不了机怎么办?"]
|
2693 |
+
passages = ["样例段落-1", "样例段落-2"]
|
2694 |
+
instruction = "为这个句子生成表示以用于检索相关文章:"
|
2695 |
+
|
2696 |
+
model = SentenceTransformer('BAAI/bge-large-zh')
|
2697 |
+
q_embeddings = model.encode([instruction+q for q in queries], normalize_embeddings=True)
|
2698 |
+
p_embeddings = model.encode(passages, normalize_embeddings=True)
|
2699 |
+
scores = q_embeddings @ p_embeddings.T
|
2700 |
+
```
|
2701 |
+
|
2702 |
+
* **Using HuggingFace Transformers**
|
2703 |
+
|
2704 |
+
With transformers package, you can use the model like this: First, you pass your input through the transformer model, then you select the last hidden state of first token (i.e., [CLS]) as the sentence embedding.
|
2705 |
+
|
2706 |
+
```python
|
2707 |
+
from transformers import AutoTokenizer, AutoModel
|
2708 |
+
import torch
|
2709 |
+
# Sentences we want sentence embeddings for
|
2710 |
+
sentences = ["样例数据-1", "样例数据-2"]
|
2711 |
+
|
2712 |
+
# Load model from HuggingFace Hub
|
2713 |
+
tokenizer = AutoTokenizer.from_pretrained('BAAI/bge-large-zh')
|
2714 |
+
model = AutoModel.from_pretrained('BAAI/bge-large-zh')
|
2715 |
+
|
2716 |
+
# Tokenize sentences
|
2717 |
+
encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
|
2718 |
+
# for retrieval task, add a instruction to query
|
2719 |
+
# encoded_input = tokenizer([instruction + q for q in queries], padding=True, truncation=True, return_tensors='pt')
|
2720 |
+
|
2721 |
+
# Compute token embeddings
|
2722 |
+
with torch.no_grad():
|
2723 |
+
model_output = model(**encoded_input)
|
2724 |
+
# Perform pooling. In this case, cls pooling.
|
2725 |
+
sentence_embeddings = model_output[0][:, 0]
|
2726 |
+
# normalize embeddings
|
2727 |
+
sentence_embeddings = torch.nn.functional.normalize(sentence_embeddings, p=2, dim=1)
|
2728 |
+
print("Sentence embeddings:", sentence_embeddings)
|
2729 |
+
```
|
2730 |
+
|
2731 |
+
|
2732 |
+
## Evaluation
|
2733 |
+
`baai-general-embedding` models achieve **state-of-the-art performance on both MTEB and C-MTEB leaderboard!**
|
2734 |
+
More details and evaluation scripts see [benchemark](benchmark/README.md).
|
2735 |
+
|
2736 |
+
- **MTEB**:
|
2737 |
+
|
2738 |
+
| Model Name | Dimension | Sequence Length | Average (56) | Retrieval (15) |Clustering (11) | Pair Classification (3) | Reranking (4) | STS (10) | Summarization (1) | Classification (12) |
|
2739 |
+
|:----:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
|
2740 |
+
| [**bge-large-en**](https://huggingface.co/BAAI/bge-large-en) | 1024 | 512 | **63.98** | **53.9** | **46.98** | 85.8 | **59.48** | 81.56 | 32.06 | **76.21** |
|
2741 |
+
| [**bge-base-en**](https://huggingface.co/BAAI/bge-base-en) | 768 | 512 | 63.36 | 53.0 | 46.32 | 85.86 | 58.7 | 81.84 | 29.27 | 75.27 |
|
2742 |
+
| [gte-large](https://huggingface.co/thenlper/gte-large) | 1024 | 512 | 63.13 | 52.22 | 46.84 | 85.00 | 59.13 | 83.35 | 31.66 | 73.33 |
|
2743 |
+
| [gte-base](https://huggingface.co/thenlper/gte-base) | 768 | 512 | 62.39 | 51.14 | 46.2 | 84.57 | 58.61 | 82.3 | 31.17 | 73.01 |
|
2744 |
+
| [e5-large-v2](https://huggingface.co/intfloat/e5-large-v2) | 1024| 512 | 62.25 | 50.56 | 44.49 | 86.03 | 56.61 | 82.05 | 30.19 | 75.24 |
|
2745 |
+
| [**bge-small-en**](https://huggingface.co/BAAI/bge-small-en) | 384 | 512 | 62.11 | 51.82 | 44.31 | 83.78 | 57.97 | 80.72 | 30.53 | 74.37 |
|
2746 |
+
| [instructor-xl](https://huggingface.co/hkunlp/instructor-xl) | 768 | 512 | 61.79 | 49.26 | 44.74 | 86.62 | 57.29 | 83.06 | 32.32 | 61.79 |
|
2747 |
+
| [e5-base-v2](https://huggingface.co/intfloat/e5-base-v2) | 768 | 512 | 61.5 | 50.29 | 43.80 | 85.73 | 55.91 | 81.05 | 30.28 | 73.84 |
|
2748 |
+
| [gte-small](https://huggingface.co/thenlper/gte-small) | 384 | 512 | 61.36 | 49.46 | 44.89 | 83.54 | 57.7 | 82.07 | 30.42 | 72.31 |
|
2749 |
+
| [text-embedding-ada-002](https://platform.openai.com/docs/guides/embeddings) | 1536 | 8192 | 60.99 | 49.25 | 45.9 | 84.89 | 56.32 | 80.97 | 30.8 | 70.93 |
|
2750 |
+
| [e5-small-v2](https://huggingface.co/intfloat/e5-base-v2) | 384 | 512 | 59.93 | 49.04 | 39.92 | 84.67 | 54.32 | 80.39 | 31.16 | 72.94 |
|
2751 |
+
| [sentence-t5-xxl](https://huggingface.co/sentence-transformers/sentence-t5-xxl) | 768 | 512 | 59.51 | 42.24 | 43.72 | 85.06 | 56.42 | 82.63 | 30.08 | 73.42 |
|
2752 |
+
| [all-mpnet-base-v2](https://huggingface.co/sentence-transformers/all-mpnet-base-v2) | 768 | 514 | 57.78 | 43.81 | 43.69 | 83.04 | 59.36 | 80.28 | 27.49 | 65.07 |
|
2753 |
+
| [sgpt-bloom-7b1-msmarco](https://huggingface.co/bigscience/sgpt-bloom-7b1-msmarco) | 4096 | 2048 | 57.59 | 48.22 | 38.93 | 81.9 | 55.65 | 77.74 | 33.6 | 66.19 |
|
2754 |
+
| [all-MiniLM-L12-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2) | 384 | 512 | 56.53 | 42.69 | 41.81 | 82.41 | 58.44 | 79.8 | 27.9 | 63.21 |
|
2755 |
+
| [all-MiniLM-L6-v2](https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2) | 384 | 512 | 56.26 | 41.95 | 42.35 | 82.37 | 58.04 | 78.9 | 30.81 | 63.05 |
|
2756 |
+
| [contriever-base-msmarco](https://huggingface.co/nthakur/contriever-base-msmarco) | 768 | 512 | 56.00 | 41.88 | 41.1 | 82.54 | 53.14 | 76.51 | 30.36 | 66.68 |
|
2757 |
+
| [sentence-t5-base](https://huggingface.co/sentence-transformers/sentence-t5-base) | 768 | 512 | 55.27 | 33.63 | 40.21 | 85.18 | 53.09 | 81.14 | 31.39 | 69.81 |
|
2758 |
+
|
2759 |
+
|
2760 |
+
|
2761 |
+
- **C-MTEB**:
|
2762 |
+
We create a benchmark C-MTEB for Chinese text embedding which consists of 31 datasets from 6 tasks.
|
2763 |
+
Please refer to [benchemark](benchmark/README.md) for a detailed introduction.
|
2764 |
+
|
2765 |
+
| Model | Embedding dimension | Avg | Retrieval | STS | PairClassification | Classification | Reranking | Clustering |
|
2766 |
+
|:-------------------------------|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|:--------:|
|
2767 |
+
| [**bge-large-zh**](https://huggingface.co/BAAI/bge-large-zh) | 1024 | **64.20** | **71.53** | **53.23** | **78.94** | 72.26 | **65.11** | 48.39 |
|
2768 |
+
| [**bge-large-zh-noinstruct**](https://huggingface.co/BAAI/bge-large-zh-noinstruct) | 1024 | 63.53 | 70.55 | 50.98 | 76.77 | **72.49** | 64.91 | **50.01** |
|
2769 |
+
| [**BAAI/bge-base-zh**](https://huggingface.co/BAAI/bge-base-zh) | 768 | 62.96 | 69.53 | 52.05 | 77.5 | 70.98 | 64.91 | 47.63 |
|
2770 |
+
| [**BAAI/bge-small-zh**](https://huggingface.co/BAAI/bge-small-zh) | 512 | 58.27 | 63.07 | 46.87 | 70.35 | 67.78 | 61.48 | 45.09 |
|
2771 |
+
| [m3e-base](https://huggingface.co/moka-ai/m3e-base) | 768 | 57.10 |56.91 | 48.15 | 63.99 | 70.28 | 59.34 | 47.68 |
|
2772 |
+
| [m3e-large](https://huggingface.co/moka-ai/m3e-large) | 1024 | 57.05 |54.75 | 48.64 | 64.3 | 71.22 | 59.66 | 48.88 |
|
2773 |
+
| [text-embedding-ada-002(OpenAI)](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings) | 1536 | 53.02 | 52.0 | 40.61 | 69.56 | 67.38 | 54.28 | 45.68 |
|
2774 |
+
| [luotuo](https://huggingface.co/silk-road/luotuo-bert-medium) | 1024 | 49.37 | 44.4 | 39.41 | 66.62 | 65.29 | 49.25 | 44.39 |
|
2775 |
+
| [text2vec](https://huggingface.co/shibing624/text2vec-base-chinese) | 768 | 47.63 | 38.79 | 41.71 | 67.41 | 65.18 | 49.45 | 37.66 |
|
2776 |
+
| [text2vec-large](https://huggingface.co/GanymedeNil/text2vec-large-chinese) | 1024 | 47.36 | 41.94 | 41.98 | 70.86 | 63.42 | 49.16 | 30.02 |
|
2777 |
+
|
2778 |
+
|
2779 |
+
|
2780 |
+
|
2781 |
+
## Train
|
2782 |
+
This section will introduce the way we used to train the general embedding.
|
2783 |
+
The training scripts are in [flag_embedding](https://github.com/FlagOpen/FlagEmbedding/tree/master/flag_embedding/baai_general_embedding/),
|
2784 |
+
and we provide some examples to do [pre-train](https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/pretrain/) and [fine-tune](https://github.com/FlagOpen/FlagEmbedding/tree/master/examples/finetune).
|
2785 |
+
|
2786 |
+
|
2787 |
+
**1. RetroMAE Pre-train**
|
2788 |
+
We pre-train the model following the method [retromae](https://github.com/staoxiao/RetroMAE),
|
2789 |
+
which shows promising improvement in retrieval task ([paper](https://aclanthology.org/2022.emnlp-main.35.pdf)).
|
2790 |
+
The pre-training was conducted on 24 A100(40G) GPUs with a batch size of 720.
|
2791 |
+
In retromae, the mask ratio of the encoder and decoder are 0.3, and 0.5 respectively.
|
2792 |
+
We used the AdamW optimizer and the learning rate is 2e-5.
|
2793 |
+
|
2794 |
+
**Pre-training data**:
|
2795 |
+
- English:
|
2796 |
+
- [Pile](https://pile.eleuther.ai/)
|
2797 |
+
- [wikipedia](https://huggingface.co/datasets/wikipedia)
|
2798 |
+
- [msmarco](https://huggingface.co/datasets/Tevatron/msmarco-passage-corpus)
|
2799 |
+
- Chinese:
|
2800 |
+
- Subset of [wudao](https://github.com/BAAI-WuDao/Data)
|
2801 |
+
- [baidu-baike](https://baike.baidu.com/)
|
2802 |
+
|
2803 |
+
|
2804 |
+
**2. Finetune**
|
2805 |
+
We fine-tune the model using a contrastive objective.
|
2806 |
+
The format of input data is a triple`(query, positive, negative)`.
|
2807 |
+
Besides the negative in the triple, we also adopt in-batch negatives strategy.
|
2808 |
+
We employ the cross-device negatives sharing method to share negatives among different GPUs,
|
2809 |
+
which can dramatically **increase the number of negatives**.
|
2810 |
+
|
2811 |
+
We trained our model on 48 A100(40G) GPUs with a large batch size of 32,768 (so there are **65,535** negatives for each query in a batch).
|
2812 |
+
We used the AdamW optimizer and the learning rate is 1e-5.
|
2813 |
+
The temperature for contrastive loss is 0.01.
|
2814 |
+
|
2815 |
+
For the version with `*-instrcution`, we add instruction to the query for the retrieval task in the training.
|
2816 |
+
For English, the instruction is `Represent this sentence for searching relevant passages: `;
|
2817 |
+
For Chinese, the instruction is `为这个句子生成表示以用于检索相关文章:`.
|
2818 |
+
In the evaluation, the instruction should be added for sentence to passages retrieval task, not be added for other tasks.
|
2819 |
+
|
2820 |
+
|
2821 |
+
The finetune script is accessible in this repository: [flag_embedding](https://github.com/FlagOpen/FlagEmbedding/tree/master/flag_embedding/baai_general_embedding/README.md).
|
2822 |
+
You can easily finetune your model with it.
|
2823 |
+
|
2824 |
+
**Training data**:
|
2825 |
+
|
2826 |
+
- For English, we collect 230M text pairs from [wikipedia](https://huggingface.co/datasets/wikipedia), [cc-net](https://github.com/facebookresearch/cc_net), and so on.
|
2827 |
+
|
2828 |
+
- For Chinese, we collect 120M text pairs from [wudao](https://github.com/BAAI-WuDao/Data), [simclue](https://github.com/CLUEbenchmark/SimCLUE) and so on.
|
2829 |
+
|
2830 |
+
**The data collection is to be released in the future.**
|
2831 |
+
|
2832 |
+
We will continually update the embedding models and training codes,
|
2833 |
+
hoping to promote the development of the embedding model community.
|
2834 |
+
|
2835 |
+
|
2836 |
+
## License
|
2837 |
+
FlagEmbedding is licensed under [MIT License](). The released models can be used for commercial purposes free of charge.
|