Sentence Transformers integration (#2)

- Add Sentence Transformers integration + README (c28c400be3e576ba1a36de41b276f00936e80cb1)

1_Pooling/config.json

@@ -0,0 +1,10 @@
+{
+  "word_embedding_dimension": 768,
+  "pooling_mode_cls_token": true,
+  "pooling_mode_mean_tokens": false,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false,
+  "pooling_mode_weightedmean_tokens": false,
+  "pooling_mode_lasttoken": false,
+  "include_prompt": true
+}

README.md

@@ -2936,6 +2936,39 @@ Based on the [intfloat/e5-large-unsupervised](https://huggingface.co/intfloat/e5
 ## Usage
 
 
+### Using Sentence Transformers
+
+You can use the sentence-transformers package to use an snowflake-arctic-embed model, as shown below. 
+
+```python
+from sentence_transformers import SentenceTransformer
+
+model = SentenceTransformer("Snowflake/snowflake-arctic-embed-m-long", trust_remote_code=True)
+
+queries = ['what is snowflake?', 'Where can I get the best tacos?']
+documents = ['The Data Cloud!', 'Mexico City of Course!']
+
+query_embeddings = model.encode(queries, prompt_name="query")
+document_embeddings = model.encode(documents)
+
+scores = query_embeddings @ document_embeddings.T
+for query, query_scores in zip(queries, scores):
+    doc_score_pairs = list(zip(documents, query_scores))
+    doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True)
+    # Output passages & scores
+    print("Query:", query)
+    for document, score in doc_score_pairs:
+        print(score, document)
+```
+```
+Query: what is snowflake?
+0.46484852 The Data Cloud!
+0.3758855 Mexico City of Course!
+Query: Where can I get the best tacos?
+0.42407742 Mexico City of Course!
+0.36740506 The Data Cloud!
+```
+
 ### Using Huggingface transformers
 
 
@@ -2948,7 +2981,7 @@ import torch
 from transformers import AutoModel, AutoTokenizer
 
 tokenizer = AutoTokenizer.from_pretrained('Snowflake/snowflake-arctic-embed-m-long')
-model = AutoModel.from_pretrained('Snowflake/snowflake-arctic-embed-m-long', add_pooling_layer=False)
+model = AutoModel.from_pretrained('Snowflake/snowflake-arctic-embed-m-long', trust_remote_code=True, add_pooling_layer=False)
 model.eval()
 
 query_prefix = 'Represent this sentence for searching relevant passages: '

config_sentence_transformers.json

@@ -0,0 +1,11 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.7.0.dev0",
+    "transformers": "4.39.3",
+    "pytorch": "2.1.0+cu121"
+  },
+  "prompts": {
+    "query": "Represent this sentence for searching relevant passages: "
+  },
+  "default_prompt_name": null
+}

modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 8192,
+  "do_lower_case": false
+}