Add Sentence Transformers support

#2
by tomaarsen HF staff - opened
1_Pooling/config.json CHANGED
@@ -1,10 +1,9 @@
1
  {
2
- "word_embedding_dimension": 768,
3
- "pooling_mode_cls_token": false,
4
- "pooling_mode_mean_tokens": true,
5
- "pooling_mode_max_tokens": false,
6
- "pooling_mode_mean_sqrt_len_tokens": false,
7
- "pooling_mode_weightedmean_tokens": false,
8
- "pooling_mode_lasttoken": false
9
- }
10
-
 
1
  {
2
+ "word_embedding_dimension": 768,
3
+ "pooling_mode_cls_token": false,
4
+ "pooling_mode_mean_tokens": true,
5
+ "pooling_mode_max_tokens": false,
6
+ "pooling_mode_mean_sqrt_len_tokens": false,
7
+ "pooling_mode_weightedmean_tokens": false,
8
+ "pooling_mode_lasttoken": false
9
+ }
 
README.md CHANGED
@@ -1,10 +1,16 @@
1
  ---
 
 
 
 
 
 
 
 
2
  license: apache-2.0
3
  language:
4
  - en
5
  inference: false
6
- tags:
7
- - mteb
8
  model-index:
9
  - name: epoch_0_model
10
  results:
@@ -2660,7 +2666,20 @@ Training data to train the models is released in its entirety. For more details,
2660
 
2661
  ## Usage
2662
 
 
 
 
 
 
 
 
 
 
 
 
 
2663
 
 
2664
  ```python
2665
  import torch
2666
  import torch.nn.functional as F
@@ -2671,7 +2690,7 @@ def mean_pooling(model_output, attention_mask):
2671
  input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
2672
  return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
2673
 
2674
- sentences = ['What is TSNE?', 'Who is Laurens van der Maaten?']
2675
 
2676
  tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
2677
  model = AutoModel.from_pretrained('nomic-ai/nomic-embed-text-v1-unsupervised', trust_remote_code=True)
 
1
  ---
2
+ library_name: sentence-transformers
3
+ pipeline_tag: sentence-similarity
4
+ tags:
5
+ - feature-extraction
6
+ - sentence-similarity
7
+ - mteb
8
+ - transformers
9
+ - transformers.js
10
  license: apache-2.0
11
  language:
12
  - en
13
  inference: false
 
 
14
  model-index:
15
  - name: epoch_0_model
16
  results:
 
2666
 
2667
  ## Usage
2668
 
2669
+ Note `nomic-embed-text` requires prefixes! We support the prefixes `[search_query, search_document, classification, clustering]`.
2670
+ For retrieval applications, you should prepend `search_document` for all your documents and `search_query` for your queries.
2671
+
2672
+ ### Sentence Transformers
2673
+ ```python
2674
+ from sentence_transformers import SentenceTransformer
2675
+
2676
+ model = SentenceTransformer("nomic-ai/nomic-embed-text-v1-unsupervised", trust_remote_code=True)
2677
+ sentences = ['search_query: What is TSNE?', 'search_query: Who is Laurens van der Maaten?']
2678
+ embeddings = model.encode(sentences)
2679
+ print(embeddings)
2680
+ ```
2681
 
2682
+ ### Transformers
2683
  ```python
2684
  import torch
2685
  import torch.nn.functional as F
 
2690
  input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
2691
  return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
2692
 
2693
+ sentences = ['search_query: What is TSNE?', 'search_query: Who is Laurens van der Maaten?']
2694
 
2695
  tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')
2696
  model = AutoModel.from_pretrained('nomic-ai/nomic-embed-text-v1-unsupervised', trust_remote_code=True)
config.json CHANGED
@@ -11,7 +11,7 @@
11
  "bos_token_id": null,
12
  "causal": false,
13
  "dense_seq_output": true,
14
- "embd_pdrop": 0.0,
15
  "eos_token_id": null,
16
  "fused_bias_fc": true,
17
  "fused_dropout_add_ln": true,
@@ -31,7 +31,7 @@
31
  "prenorm": false,
32
  "qkv_proj_bias": false,
33
  "reorder_and_upcast_attn": false,
34
- "resid_pdrop": 0.0,
35
  "rotary_emb_base": 1000,
36
  "rotary_emb_fraction": 1.0,
37
  "rotary_emb_interleaved": false,
@@ -40,7 +40,7 @@
40
  "scale_attn_by_inverse_layer_idx": false,
41
  "scale_attn_weights": true,
42
  "summary_activation": null,
43
- "summary_first_dropout": 0.0,
44
  "summary_proj_to_labels": true,
45
  "summary_type": "cls_index",
46
  "summary_use_proj": true,
 
11
  "bos_token_id": null,
12
  "causal": false,
13
  "dense_seq_output": true,
14
+ "embd_pdrop": 0.1,
15
  "eos_token_id": null,
16
  "fused_bias_fc": true,
17
  "fused_dropout_add_ln": true,
 
31
  "prenorm": false,
32
  "qkv_proj_bias": false,
33
  "reorder_and_upcast_attn": false,
34
+ "resid_pdrop": 0.1,
35
  "rotary_emb_base": 1000,
36
  "rotary_emb_fraction": 1.0,
37
  "rotary_emb_interleaved": false,
 
40
  "scale_attn_by_inverse_layer_idx": false,
41
  "scale_attn_weights": true,
42
  "summary_activation": null,
43
+ "summary_first_dropout": 0.1,
44
  "summary_proj_to_labels": true,
45
  "summary_type": "cls_index",
46
  "summary_use_proj": true,
config_sentence_transformers.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "__version__": {
3
+ "sentence_transformers": "2.4.0.dev0",
4
+ "transformers": "4.37.2",
5
+ "pytorch": "2.1.0+cu121"
6
+ }
7
+ }
modeling_hf_nomic_bert.py CHANGED
@@ -1069,6 +1069,7 @@ class NomicBertModel(NomicBertPreTrainedModel):
1069
  position_ids=None,
1070
  token_type_ids=None,
1071
  attention_mask=None,
 
1072
  ):
1073
  if token_type_ids is None:
1074
  token_type_ids = torch.zeros_like(input_ids)
@@ -1080,7 +1081,7 @@ class NomicBertModel(NomicBertPreTrainedModel):
1080
 
1081
  attention_mask = self.get_extended_attention_mask(attention_mask, input_ids.shape)
1082
  sequence_output = self.encoder(
1083
- hidden_states, attention_mask=attention_mask
1084
  )
1085
 
1086
  pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
 
1069
  position_ids=None,
1070
  token_type_ids=None,
1071
  attention_mask=None,
1072
+ return_dict=None,
1073
  ):
1074
  if token_type_ids is None:
1075
  token_type_ids = torch.zeros_like(input_ids)
 
1081
 
1082
  attention_mask = self.get_extended_attention_mask(attention_mask, input_ids.shape)
1083
  sequence_output = self.encoder(
1084
+ hidden_states, attention_mask=attention_mask, return_dict=return_dict,
1085
  )
1086
 
1087
  pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
modules.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "idx": 0,
4
+ "name": "0",
5
+ "path": "",
6
+ "type": "sentence_transformers.models.Transformer"
7
+ },
8
+ {
9
+ "idx": 1,
10
+ "name": "1",
11
+ "path": "1_Pooling",
12
+ "type": "sentence_transformers.models.Pooling"
13
+ },
14
+ {
15
+ "idx": 2,
16
+ "name": "2",
17
+ "path": "2_Normalize",
18
+ "type": "sentence_transformers.models.Normalize"
19
+ }
20
+ ]
sentence_bert_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "max_seq_length": 8192,
3
+ "do_lower_case": false
4
+ }
tokenizer_config.json CHANGED
@@ -46,7 +46,7 @@
46
  "cls_token": "[CLS]",
47
  "do_lower_case": true,
48
  "mask_token": "[MASK]",
49
- "model_max_length": 512,
50
  "pad_token": "[PAD]",
51
  "sep_token": "[SEP]",
52
  "strip_accents": null,
 
46
  "cls_token": "[CLS]",
47
  "do_lower_case": true,
48
  "mask_token": "[MASK]",
49
+ "model_max_length": 8192,
50
  "pad_token": "[PAD]",
51
  "sep_token": "[SEP]",
52
  "strip_accents": null,