Upload folder using huggingface_hub
Browse files- adapter_model.safetensors +1 -1
- poly_encoder.py +90 -0
adapter_model.safetensors
CHANGED
@@ -1,3 +1,3 @@
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
-
oid sha256:
|
3 |
size 209736952
|
|
|
1 |
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b9257294989d1901fc44650804efd5878092c143986bbb9a12120de62e4773bb
|
3 |
size 209736952
|
poly_encoder.py
ADDED
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import random
|
2 |
+
import numpy as np
|
3 |
+
import torch
|
4 |
+
from transformers import BertTokenizer, BertModel
|
5 |
+
import torch.nn.functional as F
|
6 |
+
import nltk
|
7 |
+
nltk.download('punkt')
|
8 |
+
from nltk.tokenize import sent_tokenize
|
9 |
+
|
10 |
+
def set_seed(seed):
|
11 |
+
torch.manual_seed(seed)
|
12 |
+
random.seed(seed)
|
13 |
+
np.random.seed(seed)
|
14 |
+
if torch.cuda.is_available():
|
15 |
+
torch.cuda.manual_seed_all(seed)
|
16 |
+
|
17 |
+
class PolyEncoder(torch.nn.Module):
|
18 |
+
def __init__(self, bert_model_name='klue/bert-base', poly_m=16):
|
19 |
+
super(PolyEncoder, self).__init__()
|
20 |
+
self.poly_m = poly_m
|
21 |
+
self.bert_model = BertModel.from_pretrained(bert_model_name)
|
22 |
+
self.poly_code_embeddings = torch.nn.Embedding(poly_m, self.bert_model.config.hidden_size)
|
23 |
+
|
24 |
+
def forward(self, context_input_ids, context_attention_mask, question_input_ids, question_attention_mask):
|
25 |
+
# Encode the question
|
26 |
+
question_outputs = self.bert_model(input_ids=question_input_ids, attention_mask=question_attention_mask)
|
27 |
+
question_cls_embeddings = question_outputs.last_hidden_state[:, 0, :] # CLS token
|
28 |
+
|
29 |
+
# Encode the context
|
30 |
+
context_outputs = self.bert_model(input_ids=context_input_ids, attention_mask=context_attention_mask)
|
31 |
+
context_hidden_states = context_outputs.last_hidden_state
|
32 |
+
|
33 |
+
# Poly codes
|
34 |
+
poly_codes = self.poly_code_embeddings.weight.unsqueeze(0).expand(context_hidden_states.size(0), -1, -1)
|
35 |
+
|
36 |
+
# Context and poly code interactions
|
37 |
+
attention_weights = F.softmax(torch.einsum('bmd,bnd->bmn', context_hidden_states, poly_codes), dim=-1)
|
38 |
+
poly_context_embeddings = torch.einsum('bmn,bmd->bnd', attention_weights, context_hidden_states)
|
39 |
+
|
40 |
+
# Question and poly context interactions
|
41 |
+
scores = torch.einsum('bnd,bmd->bnm', poly_context_embeddings, question_cls_embeddings.unsqueeze(1).expand(-1, self.poly_m, -1))
|
42 |
+
|
43 |
+
# Aggregate scores over poly_m dimension
|
44 |
+
scores = scores.max(dim=1).values
|
45 |
+
|
46 |
+
return scores
|
47 |
+
|
48 |
+
def get_top_n_relevant_sentences(context, question, tokenizer, model, top_n):
|
49 |
+
context_sentences = sent_tokenize(context) # NLTK๋ฅผ ์ฌ์ฉํ์ฌ ๋ฌธ์ฅ ๋ถํ
|
50 |
+
|
51 |
+
context_inputs = tokenizer(context_sentences, padding=True, truncation=True, return_tensors='pt')
|
52 |
+
question_inputs = tokenizer(question, return_tensors='pt')
|
53 |
+
|
54 |
+
with torch.no_grad():
|
55 |
+
scores = model(context_inputs['input_ids'], context_inputs['attention_mask'],
|
56 |
+
question_inputs['input_ids'].expand(len(context_sentences), -1),
|
57 |
+
question_inputs['attention_mask'].expand(len(context_sentences), -1))
|
58 |
+
|
59 |
+
score_rows, score_cols = scores.shape
|
60 |
+
|
61 |
+
scores_index = scores[:, 0].tolist()
|
62 |
+
indexed_dict = {idx: value for idx, value in enumerate(scores_index)}
|
63 |
+
sorted_dict = dict(sorted(indexed_dict.items(), key=lambda item: item[1], reverse=True))
|
64 |
+
sorted_data = sorted(sorted_dict.items(), key=lambda item: item[1], reverse=True)
|
65 |
+
top_n_keys = list(sorted_dict.keys())[:top_n]
|
66 |
+
unique_values = set()
|
67 |
+
top_keys = []
|
68 |
+
|
69 |
+
for key, value in sorted_data:
|
70 |
+
if value not in unique_values:
|
71 |
+
unique_values.add(value)
|
72 |
+
top_keys.append(key)
|
73 |
+
if len(top_keys) == top_n:
|
74 |
+
break
|
75 |
+
|
76 |
+
top_n_sentences = [context_sentences[idx] for idx in top_keys]
|
77 |
+
return top_n_sentences
|
78 |
+
|
79 |
+
# ์์ ์คํ ํจ์
|
80 |
+
def run_example(context, question):
|
81 |
+
# ๋ชจ๋ธ ๋ฐ ํ ํฌ๋์ด์ ๋ก๋๋ฅผ ์ ์ญ ๋ณ์๋ก ์ค์
|
82 |
+
tokenizer = BertTokenizer.from_pretrained('klue/bert-base')
|
83 |
+
model = PolyEncoder(bert_model_name='klue/bert-base')
|
84 |
+
|
85 |
+
top_n_sentences = get_top_n_relevant_sentences(context, question, tokenizer, model, top_n=5)
|
86 |
+
sentences = ""
|
87 |
+
for sentence in top_n_sentences:
|
88 |
+
sentences+=sentence
|
89 |
+
print(sentences)
|
90 |
+
return sentences
|