ZoeYou/QatentBert-CPC

How to create a sentence embedding using this model?

Usage (Sentence-Transformers)

import os
from transformers import BertModel, BertTokenizer
from sentence_transformers import SentenceTransformer, models

# Step 1: Load your custom pretrained model and tokenizer
model_name = "ZoeYou/QatentBert-CPC"

# Create a directory to save your model and tokenizer
model_save_path = "./custom_bert_model"
os.makedirs(model_save_path, exist_ok=True)

# Load and save your custom model and tokenizer
tokenizer = BertTokenizer.from_pretrained(model_name)
bert_model = BertModel.from_pretrained(model_name)
tokenizer.save_pretrained(model_save_path)
bert_model.save_pretrained(model_save_path)

# Step 2: Create a Transformer model using your saved custom model
word_embedding_model = models.Transformer(model_name_or_path=model_save_path)

# Step 3: Add a pooling layer with CLS pooling
pooling_model = models.Pooling(
    word_embedding_model.get_word_embedding_dimension(),
    pooling_mode_mean_tokens=False,
    pooling_mode_cls_token=True,    # Use CLS token pooling
    pooling_mode_max_tokens=False
)

# Step 4: Combine the modules into a SentenceTransformer model
sentence_model = SentenceTransformer(modules=[word_embedding_model, pooling_model])

# Step 5: Encode sentences or documents
sentences = ["This is an example sentence.", "Here is another one."]
sentence_embeddings = sentence_model.encode(sentences)

print(sentence_embeddings.shape)  # Should be (batch_size, embedding_dim)