NLP2024_title_search / prepare_pool.py
kaisugi's picture
initial commit
3f9f23d
raw history blame
No virus
1.53 kB
import torch.nn.functional as F
import torch
from torch import Tensor
from transformers import AutoTokenizer, AutoModel
import numpy as np
import pandas as pd
def average_pool(last_hidden_states: Tensor,
attention_mask: Tensor) -> Tensor:
last_hidden = last_hidden_states.masked_fill(~attention_mask[..., None].bool(), 0.0)
return last_hidden.sum(dim=1) / attention_mask.sum(dim=1)[..., None]
paper_df = pd.read_csv('anlp2024.tsv', names=["pid", "title"], sep="\t")
assert len(paper_df) == 599
# paper_df ใฎ title ๅˆ—ใซใ‚ใ‚‹ใƒ†ใ‚ญใ‚นใƒˆใ‚’ใƒชใ‚นใƒˆใซๅค‰ๆ›ใ—ใŸไธŠใงใ€ๅ„ๆ–‡ๅญ—ๅˆ—ใฎๆˆฆ้—˜ใซ "passage: " ใ‚’ใใ‚Œใžใ‚Œไป˜ใ‘ๅŠ ใˆใฆ input_texts ใจใ™ใ‚‹
input_texts = [f"passage: {title}" for title in paper_df["title"].tolist()]
assert input_texts[0] == "passage: ๅธ‚ๆณใ‚ณใƒกใƒณใƒˆ็”ŸๆˆใฎใŸใ‚ใฎๅฐ‘ๆ•ฐไบ‹ไพ‹้ธๆŠž"
assert input_texts[-1] == "passage: Event-Centered Prompting for Text Style Transfer"
tokenizer = AutoTokenizer.from_pretrained('intfloat/multilingual-e5-large')
model = AutoModel.from_pretrained('intfloat/multilingual-e5-large')
# Tokenize the input texts
batch_dict = tokenizer(input_texts, max_length=512, padding=True, truncation=True, return_tensors='pt')
with torch.no_grad():
outputs = model(**batch_dict)
embeddings = average_pool(outputs.last_hidden_state, batch_dict['attention_mask'])
embeddings = F.normalize(embeddings, p=2, dim=1)
assert embeddings.shape == (599, 1024)
np.savez("anlp2024", embeddings.detach().numpy().copy())