Add Sentence Transformers support

#2
by tomaarsen HF staff - opened
1_Pooling/config.json CHANGED
@@ -1,10 +1,9 @@
1
  {
2
- "word_embedding_dimension": 768,
3
- "pooling_mode_cls_token": false,
4
- "pooling_mode_mean_tokens": true,
5
- "pooling_mode_max_tokens": false,
6
- "pooling_mode_mean_sqrt_len_tokens": false,
7
- "pooling_mode_weightedmean_tokens": false,
8
- "pooling_mode_lasttoken": false
9
- }
10
-
 
1
  {
2
+ "word_embedding_dimension": 768,
3
+ "pooling_mode_cls_token": false,
4
+ "pooling_mode_mean_tokens": true,
5
+ "pooling_mode_max_tokens": false,
6
+ "pooling_mode_mean_sqrt_len_tokens": false,
7
+ "pooling_mode_weightedmean_tokens": false,
8
+ "pooling_mode_lasttoken": false
9
+ }
 
README.md CHANGED
@@ -1,6 +1,10 @@
1
  ---
 
 
2
  tags:
3
- - mteb
 
 
4
  model-index:
5
  - name: epoch_0_model
6
  results:
@@ -2653,6 +2657,17 @@ Training data to train the models is released in its entirety. For more details,
2653
 
2654
  ## Usage
2655
 
 
 
 
 
 
 
 
 
 
 
 
2656
 
2657
  ```python
2658
  import torch
 
1
  ---
2
+ library_name: sentence-transformers
3
+ pipeline_tag: sentence-similarity
4
  tags:
5
+ - feature-extraction
6
+ - sentence-similarity
7
+ - mteb
8
  model-index:
9
  - name: epoch_0_model
10
  results:
 
2657
 
2658
  ## Usage
2659
 
2660
+ ### Sentence Transformers
2661
+ ```python
2662
+ from sentence_transformers import SentenceTransformer
2663
+
2664
+ model = SentenceTransformer("nomic-ai/nomic-embed-text-v1-ablated", trust_remote_code=True)
2665
+ sentences = ['What is TSNE?', 'Who is Laurens van der Maaten?']
2666
+ embeddings = model.encode(sentences)
2667
+ print(embeddings)
2668
+ ```
2669
+
2670
+ ### Transformers
2671
 
2672
  ```python
2673
  import torch
config_sentence_transformers.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "__version__": {
3
+ "sentence_transformers": "2.4.0.dev0",
4
+ "transformers": "4.37.2",
5
+ "pytorch": "2.1.0+cu121"
6
+ }
7
+ }
modeling_hf_nomic_bert.py CHANGED
@@ -1069,6 +1069,7 @@ class NomicBertModel(NomicBertPreTrainedModel):
1069
  position_ids=None,
1070
  token_type_ids=None,
1071
  attention_mask=None,
 
1072
  ):
1073
  if token_type_ids is None:
1074
  token_type_ids = torch.zeros_like(input_ids)
@@ -1080,7 +1081,7 @@ class NomicBertModel(NomicBertPreTrainedModel):
1080
 
1081
  attention_mask = self.get_extended_attention_mask(attention_mask, input_ids.shape)
1082
  sequence_output = self.encoder(
1083
- hidden_states, attention_mask=attention_mask
1084
  )
1085
 
1086
  pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
 
1069
  position_ids=None,
1070
  token_type_ids=None,
1071
  attention_mask=None,
1072
+ return_dict=None,
1073
  ):
1074
  if token_type_ids is None:
1075
  token_type_ids = torch.zeros_like(input_ids)
 
1081
 
1082
  attention_mask = self.get_extended_attention_mask(attention_mask, input_ids.shape)
1083
  sequence_output = self.encoder(
1084
+ hidden_states, attention_mask=attention_mask, return_dict=return_dict,
1085
  )
1086
 
1087
  pooled_output = self.pooler(sequence_output) if self.pooler is not None else None
modules.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [
2
+ {
3
+ "idx": 0,
4
+ "name": "0",
5
+ "path": "",
6
+ "type": "sentence_transformers.models.Transformer"
7
+ },
8
+ {
9
+ "idx": 1,
10
+ "name": "1",
11
+ "path": "1_Pooling",
12
+ "type": "sentence_transformers.models.Pooling"
13
+ },
14
+ {
15
+ "idx": 2,
16
+ "name": "2",
17
+ "path": "2_Normalize",
18
+ "type": "sentence_transformers.models.Normalize"
19
+ }
20
+ ]
sentence_bert_config.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "max_seq_length": 8192,
3
+ "do_lower_case": false
4
+ }
tokenizer_config.json CHANGED
@@ -46,7 +46,7 @@
46
  "cls_token": "[CLS]",
47
  "do_lower_case": true,
48
  "mask_token": "[MASK]",
49
- "model_max_length": 512,
50
  "pad_token": "[PAD]",
51
  "sep_token": "[SEP]",
52
  "strip_accents": null,
 
46
  "cls_token": "[CLS]",
47
  "do_lower_case": true,
48
  "mask_token": "[MASK]",
49
+ "model_max_length": 8192,
50
  "pad_token": "[PAD]",
51
  "sep_token": "[SEP]",
52
  "strip_accents": null,