metadata

base_model: BAAI/bge-small-en-v1.5
datasets: []
language: []
library_name: sentence-transformers
metrics:
  - cosine_accuracy@1
  - cosine_accuracy@3
  - cosine_accuracy@5
  - cosine_accuracy@10
  - cosine_precision@1
  - cosine_precision@3
  - cosine_precision@5
  - cosine_precision@10
  - cosine_recall@1
  - cosine_recall@3
  - cosine_recall@5
  - cosine_recall@10
  - cosine_ndcg@10
  - cosine_mrr@10
  - cosine_map@100
pipeline_tag: sentence-similarity
tags:
  - sentence-transformers
  - sentence-similarity
  - feature-extraction
  - generated_from_trainer
  - dataset_size:60323
  - loss:MatryoshkaLoss
  - loss:MultipleNegativesRankingLoss
widget:
  - source_sentence: No recipes found with these beef stock powder and orange juice!
    sentences:
      - Can you provide recipe ideas with beef stock powder and orange juice?
      - >-
        What are some recipes that utilize jasmine rice and thai red curry paste
        effectively?
      - What recipes incorporate broccoli and bacon into meals?
  - source_sentence: No recipes found with these nutmeg flower and angel hair rice noodles!
    sentences:
      - What dishes can be created with kale and bok choy?
      - >-
        What recipes incorporate green zucchini and vegan ground beef into
        meals?
      - >-
        Can you provide me with meal ideas using nutmeg flower and angel hair
        rice noodles?
  - source_sentence: No recipes found with these cinnamon and ground lamb!
    sentences:
      - Can you suggest dishes where cinnamon and ground lamb is key?
      - What diet tags are relevant to Sneha's Aloo Baingan ?
      - What recipes are there with toasted sesame oil and red lentils/masoor?
  - source_sentence: No recipes found with these red lentils/masoor and bok choy!
    sentences:
      - What are the culinary uses of chili sauce and sriracha?
      - What are some ways to use canned tomato puree and frozen ube in recipes?
      - What are some ideas for dishes with red lentils/masoor and bok choy?
  - source_sentence: No recipes found with these red onion and cubed stuffing!
    sentences:
      - >-
        Can you provide meal suggestions involving vanilla extract and brown
        lentil/black masoor dal?
      - >-
        What recipes incorporate methi (fenugreek) and honey in their
        ingredients?
      - >-
        What culinary preparations can be made with red onion and cubed
        stuffing?
model-index:
  - name: SentenceTransformer based on BAAI/bge-small-en-v1.5
    results:
      - task:
          type: information-retrieval
          name: Information Retrieval
        dataset:
          name: dim 384
          type: dim_384
        metrics:
          - type: cosine_accuracy@1
            value: 0.9819483813217962
            name: Cosine Accuracy@1
          - type: cosine_accuracy@3
            value: 0.9976130091004028
            name: Cosine Accuracy@3
          - type: cosine_accuracy@5
            value: 0.9995524392063255
            name: Cosine Accuracy@5
          - type: cosine_accuracy@10
            value: 1
            name: Cosine Accuracy@10
          - type: cosine_precision@1
            value: 0.9819483813217962
            name: Cosine Precision@1
          - type: cosine_precision@3
            value: 0.33253766970013426
            name: Cosine Precision@3
          - type: cosine_precision@5
            value: 0.1999104878412651
            name: Cosine Precision@5
          - type: cosine_precision@10
            value: 0.09999999999999999
            name: Cosine Precision@10
          - type: cosine_recall@1
            value: 0.9819483813217962
            name: Cosine Recall@1
          - type: cosine_recall@3
            value: 0.9976130091004028
            name: Cosine Recall@3
          - type: cosine_recall@5
            value: 0.9995524392063255
            name: Cosine Recall@5
          - type: cosine_recall@10
            value: 1
            name: Cosine Recall@10
          - type: cosine_ndcg@10
            value: 0.9923670621371893
            name: Cosine Ndcg@10
          - type: cosine_mrr@10
            value: 0.9897597379993318
            name: Cosine Mrr@10
          - type: cosine_map@100
            value: 0.9897597379993323
            name: Cosine Map@100
      - task:
          type: information-retrieval
          name: Information Retrieval
        dataset:
          name: dim 256
          type: dim_256
        metrics:
          - type: cosine_accuracy@1
            value: 0.9812024466656721
            name: Cosine Accuracy@1
          - type: cosine_accuracy@3
            value: 0.997463822169178
            name: Cosine Accuracy@3
          - type: cosine_accuracy@5
            value: 0.9998508130687752
            name: Cosine Accuracy@5
          - type: cosine_accuracy@10
            value: 1
            name: Cosine Accuracy@10
          - type: cosine_precision@1
            value: 0.9812024466656721
            name: Cosine Precision@1
          - type: cosine_precision@3
            value: 0.3324879407230593
            name: Cosine Precision@3
          - type: cosine_precision@5
            value: 0.19997016261375503
            name: Cosine Precision@5
          - type: cosine_precision@10
            value: 0.09999999999999999
            name: Cosine Precision@10
          - type: cosine_recall@1
            value: 0.9812024466656721
            name: Cosine Recall@1
          - type: cosine_recall@3
            value: 0.997463822169178
            name: Cosine Recall@3
          - type: cosine_recall@5
            value: 0.9998508130687752
            name: Cosine Recall@5
          - type: cosine_recall@10
            value: 1
            name: Cosine Recall@10
          - type: cosine_ndcg@10
            value: 0.9921395779775503
            name: Cosine Ndcg@10
          - type: cosine_mrr@10
            value: 0.9894450246158434
            name: Cosine Mrr@10
          - type: cosine_map@100
            value: 0.9894450246158436
            name: Cosine Map@100
      - task:
          type: information-retrieval
          name: Information Retrieval
        dataset:
          name: dim 128
          type: dim_128
        metrics:
          - type: cosine_accuracy@1
            value: 0.979561390422199
            name: Cosine Accuracy@1
          - type: cosine_accuracy@3
            value: 0.9970162613755035
            name: Cosine Accuracy@3
          - type: cosine_accuracy@5
            value: 0.9998508130687752
            name: Cosine Accuracy@5
          - type: cosine_accuracy@10
            value: 1
            name: Cosine Accuracy@10
          - type: cosine_precision@1
            value: 0.979561390422199
            name: Cosine Precision@1
          - type: cosine_precision@3
            value: 0.3323387537918345
            name: Cosine Precision@3
          - type: cosine_precision@5
            value: 0.19997016261375505
            name: Cosine Precision@5
          - type: cosine_precision@10
            value: 0.09999999999999999
            name: Cosine Precision@10
          - type: cosine_recall@1
            value: 0.979561390422199
            name: Cosine Recall@1
          - type: cosine_recall@3
            value: 0.9970162613755035
            name: Cosine Recall@3
          - type: cosine_recall@5
            value: 0.9998508130687752
            name: Cosine Recall@5
          - type: cosine_recall@10
            value: 1
            name: Cosine Recall@10
          - type: cosine_ndcg@10
            value: 0.9913010184783637
            name: Cosine Ndcg@10
          - type: cosine_mrr@10
            value: 0.9883310955293644
            name: Cosine Mrr@10
          - type: cosine_map@100
            value: 0.9883310955293649
            name: Cosine Map@100
      - task:
          type: information-retrieval
          name: Information Retrieval
        dataset:
          name: dim 64
          type: dim_64
        metrics:
          - type: cosine_accuracy@1
            value: 0.9816500074593466
            name: Cosine Accuracy@1
          - type: cosine_accuracy@3
            value: 0.9968670744442787
            name: Cosine Accuracy@3
          - type: cosine_accuracy@5
            value: 0.9997016261375503
            name: Cosine Accuracy@5
          - type: cosine_accuracy@10
            value: 1
            name: Cosine Accuracy@10
          - type: cosine_precision@1
            value: 0.9816500074593466
            name: Cosine Precision@1
          - type: cosine_precision@3
            value: 0.3322890248147595
            name: Cosine Precision@3
          - type: cosine_precision@5
            value: 0.19994032522751004
            name: Cosine Precision@5
          - type: cosine_precision@10
            value: 0.09999999999999999
            name: Cosine Precision@10
          - type: cosine_recall@1
            value: 0.9816500074593466
            name: Cosine Recall@1
          - type: cosine_recall@3
            value: 0.9968670744442787
            name: Cosine Recall@3
          - type: cosine_recall@5
            value: 0.9997016261375503
            name: Cosine Recall@5
          - type: cosine_recall@10
            value: 1
            name: Cosine Recall@10
          - type: cosine_ndcg@10
            value: 0.9920343842432707
            name: Cosine Ndcg@10
          - type: cosine_mrr@10
            value: 0.9893333120209138
            name: Cosine Mrr@10
          - type: cosine_map@100
            value: 0.9893333120209146
            name: Cosine Map@100

SentenceTransformer based on BAAI/bge-small-en-v1.5

This is a sentence-transformers model finetuned from BAAI/bge-small-en-v1.5. It maps sentences & paragraphs to a 384-dimensional dense vector space and can be used for semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more.

Model Details

Model Description

Model Type: Sentence Transformer
Base model: BAAI/bge-small-en-v1.5
Maximum Sequence Length: 512 tokens
Output Dimensionality: 384 tokens
Similarity Function: Cosine Similarity

Model Sources

Documentation: Sentence Transformers Documentation
Repository: Sentence Transformers on GitHub
Hugging Face: Sentence Transformers on Hugging Face

Full Model Architecture

SentenceTransformer(
  (0): Transformer({'max_seq_length': 512, 'do_lower_case': True}) with Transformer model: BertModel 
  (1): Pooling({'word_embedding_dimension': 384, 'pooling_mode_cls_token': True, 'pooling_mode_mean_tokens': False, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False, 'pooling_mode_weightedmean_tokens': False, 'pooling_mode_lasttoken': False, 'include_prompt': True})
  (2): Normalize()
)

Usage

Direct Usage (Sentence Transformers)

First install the Sentence Transformers library:

pip install -U sentence-transformers

Then you can load this model and run inference.

from sentence_transformers import SentenceTransformer

# Download from the 🤗 Hub
model = SentenceTransformer("Adi-0-0-Gupta/Embedding")
# Run inference
sentences = [
    'No recipes found with these red onion and cubed stuffing!',
    'What culinary preparations can be made with red onion and cubed stuffing?',
    'Can you provide meal suggestions involving vanilla extract and brown lentil/black masoor dal?',
]
embeddings = model.encode(sentences)
print(embeddings.shape)
# [3, 384]

# Get the similarity scores for the embeddings
similarities = model.similarity(embeddings, embeddings)
print(similarities.shape)
# [3, 3]

Evaluation

Metrics

Information Retrieval

Dataset: dim_384
Evaluated with InformationRetrievalEvaluator

Metric	Value
cosine_accuracy@1	0.9819
cosine_accuracy@3	0.9976
cosine_accuracy@5	0.9996
cosine_accuracy@10	1.0
cosine_precision@1	0.9819
cosine_precision@3	0.3325
cosine_precision@5	0.1999
cosine_precision@10	0.1
cosine_recall@1	0.9819
cosine_recall@3	0.9976
cosine_recall@5	0.9996
cosine_recall@10	1.0
cosine_ndcg@10	0.9924
cosine_mrr@10	0.9898
cosine_map@100	0.9898

Information Retrieval

Dataset: dim_256
Evaluated with InformationRetrievalEvaluator

Metric	Value
cosine_accuracy@1	0.9812
cosine_accuracy@3	0.9975
cosine_accuracy@5	0.9999
cosine_accuracy@10	1.0
cosine_precision@1	0.9812
cosine_precision@3	0.3325
cosine_precision@5	0.2
cosine_precision@10	0.1
cosine_recall@1	0.9812
cosine_recall@3	0.9975
cosine_recall@5	0.9999
cosine_recall@10	1.0
cosine_ndcg@10	0.9921
cosine_mrr@10	0.9894
cosine_map@100	0.9894

Information Retrieval

Dataset: dim_128
Evaluated with InformationRetrievalEvaluator

Metric	Value
cosine_accuracy@1	0.9796
cosine_accuracy@3	0.997
cosine_accuracy@5	0.9999
cosine_accuracy@10	1.0
cosine_precision@1	0.9796
cosine_precision@3	0.3323
cosine_precision@5	0.2
cosine_precision@10	0.1
cosine_recall@1	0.9796
cosine_recall@3	0.997
cosine_recall@5	0.9999
cosine_recall@10	1.0
cosine_ndcg@10	0.9913
cosine_mrr@10	0.9883
cosine_map@100	0.9883

Information Retrieval

Dataset: dim_64
Evaluated with InformationRetrievalEvaluator

Metric	Value
cosine_accuracy@1	0.9817
cosine_accuracy@3	0.9969
cosine_accuracy@5	0.9997
cosine_accuracy@10	1.0
cosine_precision@1	0.9817
cosine_precision@3	0.3323
cosine_precision@5	0.1999
cosine_precision@10	0.1
cosine_recall@1	0.9817
cosine_recall@3	0.9969
cosine_recall@5	0.9997
cosine_recall@10	1.0
cosine_ndcg@10	0.992
cosine_mrr@10	0.9893
cosine_map@100	0.9893

Training Details

Training Dataset

Unnamed Dataset

Size: 60,323 training samples
Columns: positive and anchor
Approximate statistics based on the first 1000 samples:
positive anchor
type string string
details
min: 11 tokens
mean: 21.41 tokens
max: 503 tokens

min: 10 tokens
mean: 16.8 tokens
max: 31 tokens

	positive	anchor
type	string	string
details	min: 11 tokens mean: 21.41 tokens max: 503 tokens	min: 10 tokens mean: 16.8 tokens max: 31 tokens

Samples:

positive	anchor
`No recipes found with these indian cottage cheese (paneer) and bitter melon!`	`What are some culinary options with indian cottage cheese (paneer) and bitter melon?`
`No recipes found with these curry leaf and rice cakes!`	`What recipes can be made using curry leaf and rice cakes?`
`No recipes found with these bacon and rosemary!`	`What are the different culinary recipes that use bacon and rosemary?`

Loss: MatryoshkaLoss with these parameters:

{
    "loss": "MultipleNegativesRankingLoss",
    "matryoshka_dims": [
        384,
        256,
        128,
        64
    ],
    "matryoshka_weights": [
        1,
        1,
        1,
        1
    ],
    "n_dims_per_step": -1
}

Training Hyperparameters

Non-Default Hyperparameters

eval_strategy: epoch
per_device_train_batch_size: 64
per_device_eval_batch_size: 64
gradient_accumulation_steps: 8
learning_rate: 2e-05
num_train_epochs: 10
lr_scheduler_type: cosine
warmup_ratio: 0.1
bf16: True
tf32: True
load_best_model_at_end: True
optim: adamw_torch_fused
batch_sampler: no_duplicates

All Hyperparameters

Click to expand

overwrite_output_dir: False
do_predict: False
eval_strategy: epoch
prediction_loss_only: True
per_device_train_batch_size: 64
per_device_eval_batch_size: 64
per_gpu_train_batch_size: None
per_gpu_eval_batch_size: None
gradient_accumulation_steps: 8
eval_accumulation_steps: None
learning_rate: 2e-05
weight_decay: 0.0
adam_beta1: 0.9
adam_beta2: 0.999
adam_epsilon: 1e-08
max_grad_norm: 1.0
num_train_epochs: 10
max_steps: -1
lr_scheduler_type: cosine
lr_scheduler_kwargs: {}
warmup_ratio: 0.1
warmup_steps: 0
log_level: passive
log_level_replica: warning
log_on_each_node: True
logging_nan_inf_filter: True
save_safetensors: True
save_on_each_node: False
save_only_model: False
restore_callback_states_from_checkpoint: False
no_cuda: False
use_cpu: False
use_mps_device: False
seed: 42
data_seed: None
jit_mode_eval: False
use_ipex: False
bf16: True
fp16: False
fp16_opt_level: O1
half_precision_backend: auto
bf16_full_eval: False
fp16_full_eval: False
tf32: True
local_rank: 0
ddp_backend: None
tpu_num_cores: None
tpu_metrics_debug: False
debug: []
dataloader_drop_last: False
dataloader_num_workers: 0
dataloader_prefetch_factor: None
past_index: -1
disable_tqdm: False
remove_unused_columns: True
label_names: None
load_best_model_at_end: True
ignore_data_skip: False
fsdp: []
fsdp_min_num_params: 0
fsdp_config: {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}
fsdp_transformer_layer_cls_to_wrap: None
accelerator_config: {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}
deepspeed: None
label_smoothing_factor: 0.0
optim: adamw_torch_fused
optim_args: None
adafactor: False
group_by_length: False
length_column_name: length
ddp_find_unused_parameters: None
ddp_bucket_cap_mb: None
ddp_broadcast_buffers: False
dataloader_pin_memory: True
dataloader_persistent_workers: False
skip_memory_metrics: True
use_legacy_prediction_loop: False
push_to_hub: False
resume_from_checkpoint: None
hub_model_id: None
hub_strategy: every_save
hub_private_repo: False
hub_always_push: False
gradient_checkpointing: False
gradient_checkpointing_kwargs: None
include_inputs_for_metrics: False
eval_do_concat_batches: True
fp16_backend: auto
push_to_hub_model_id: None
push_to_hub_organization: None
mp_parameters:
auto_find_batch_size: False
full_determinism: False
torchdynamo: None
ray_scope: last
ddp_timeout: 1800
torch_compile: False
torch_compile_backend: None
torch_compile_mode: None
dispatch_batches: None
split_batches: None
include_tokens_per_second: False
include_num_input_tokens_seen: False
neftune_noise_alpha: None
optim_target_modules: None
batch_eval_metrics: False
batch_sampler: no_duplicates
multi_dataset_batch_sampler: proportional

Training Logs

Epoch	Step	Training Loss	dim_128_cosine_map@100	dim_256_cosine_map@100	dim_384_cosine_map@100	dim_64_cosine_map@100
0.0848	10	3.9258	-	-	-	-
0.1697	20	3.0513	-	-	-	-
0.2545	30	1.6368	-	-	-	-
0.3393	40	0.5491	-	-	-	-
0.4242	50	0.1541	-	-	-	-
0.5090	60	0.0615	-	-	-	-
0.5938	70	0.0426	-	-	-	-
0.6787	80	0.037	-	-	-	-
0.7635	90	0.0312	-	-	-	-
0.8484	100	0.0246	-	-	-	-
0.9332	110	0.029	-	-	-	-
0.9926	117	-	0.9855	0.9869	0.9869	0.9855
1.0180	120	0.0205	-	-	-	-
1.1029	130	0.0212	-	-	-	-
1.1877	140	0.0196	-	-	-	-
1.2725	150	0.0157	-	-	-	-
1.3574	160	0.0174	-	-	-	-
1.4422	170	0.0152	-	-	-	-
1.5270	180	0.0155	-	-	-	-
1.6119	190	0.0133	-	-	-	-
1.6967	200	0.0173	-	-	-	-
1.7815	210	0.014	-	-	-	-
1.8664	220	0.0127	-	-	-	-
1.9512	230	0.0116	-	-	-	-
1.9936	235	-	0.9883	0.9894	0.9898	0.9893

Framework Versions

Python: 3.10.12
Sentence Transformers: 3.0.1
Transformers: 4.41.2
PyTorch: 2.1.2+cu121
Accelerate: 0.31.0
Datasets: 2.19.1
Tokenizers: 0.19.1

Citation

BibTeX

Sentence Transformers

@inproceedings{reimers-2019-sentence-bert,
    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
    author = "Reimers, Nils and Gurevych, Iryna",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
    month = "11",
    year = "2019",
    publisher = "Association for Computational Linguistics",
    url = "https://arxiv.org/abs/1908.10084",
}

MatryoshkaLoss

@misc{kusupati2024matryoshka,
    title={Matryoshka Representation Learning}, 
    author={Aditya Kusupati and Gantavya Bhatt and Aniket Rege and Matthew Wallingford and Aditya Sinha and Vivek Ramanujan and William Howard-Snyder and Kaifeng Chen and Sham Kakade and Prateek Jain and Ali Farhadi},
    year={2024},
    eprint={2205.13147},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}

MultipleNegativesRankingLoss

@misc{henderson2017efficient,
    title={Efficient Natural Language Response Suggestion for Smart Reply}, 
    author={Matthew Henderson and Rami Al-Rfou and Brian Strope and Yun-hsuan Sung and Laszlo Lukacs and Ruiqi Guo and Sanjiv Kumar and Balint Miklos and Ray Kurzweil},
    year={2017},
    eprint={1705.00652},
    archivePrefix={arXiv},
    primaryClass={cs.CL}
}