| #!/usr/bin/env python3 | |
| from doctest import OutputChecker | |
| import sys | |
| import argparse | |
| #import torch | |
| import re | |
| import os | |
| import gradio as gr | |
| import requests | |
| from sentence_transformers import SentenceTransformer, util | |
| import torch | |
| from transformers import GPT2Tokenizer, GPT2LMHeadModel | |
| from transformers import T5Tokenizer, AutoModelForCausalLM | |
| import torch | |
| from doctest import OutputChecker | |
| import sys | |
| import torch | |
| import re | |
| import os | |
| import gradio as gr | |
| import requests | |
| import torch | |
| from transformers import GPT2Tokenizer, GPT2LMHeadModel | |
| from torch.nn.functional import softmax | |
| import numpy as np | |
| from transformers import BertJapaneseTokenizer, BertModel | |
| import torch | |
| class SentenceBertJapanese: | |
| def __init__(self, model_name_or_path, device=None): | |
| self.tokenizer = BertJapaneseTokenizer.from_pretrained(model_name_or_path) | |
| self.model = BertModel.from_pretrained(model_name_or_path) | |
| self.model.eval() | |
| if device is None: | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| self.device = torch.device(device) | |
| self.model.to(device) | |
| def _mean_pooling(self, model_output, attention_mask): | |
| token_embeddings = model_output[0] #First element of model_output contains all token embeddings | |
| input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() | |
| return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9) | |
| def encode(self, sentences, batch_size=8): | |
| all_embeddings = [] | |
| iterator = range(0, len(sentences), batch_size) | |
| for batch_idx in iterator: | |
| batch = sentences[batch_idx:batch_idx + batch_size] | |
| encoded_input = self.tokenizer.batch_encode_plus(batch, padding="longest", | |
| truncation=True, return_tensors="pt").to(self.device) | |
| model_output = self.model(**encoded_input) | |
| sentence_embeddings = self._mean_pooling(model_output, encoded_input["attention_mask"]).to('cpu') | |
| all_embeddings.extend(sentence_embeddings) | |
| # return torch.stack(all_embeddings).numpy() | |
| return torch.stack(all_embeddings) | |
| #model_sbert = SentenceTransformer('stsb-distilbert-base') | |
| model_sbert = SentenceTransformer("colorfulscoop/sbert-base-ja") | |
| #MODEL_NAME = "sonoisa/sentence-bert-base-ja-mean-tokens-v2" | |
| #model_sbert = SentenceBertJapanese(MODEL_NAME) | |
| #batch_size = 1 | |
| #scorer = LMScorer.from_pretrained('gpt2' , device=device, batch_size=batch_size) | |
| #import torch | |
| from transformers import GPT2Tokenizer, GPT2LMHeadModel | |
| import numpy as np | |
| import re | |
| # def Sort_Tuple(tup): | |
| # # (Sorts in descending order) | |
| # tup.sort(key = lambda x: x[1]) | |
| # return tup[::-1] | |
| # def softmax(x): | |
| # exps = np.exp(x) | |
| # return np.divide(exps, np.sum(exps)) | |
| # Load pre-trained model | |
| #model = GPT2LMHeadModel.from_pretrained('distilgpt2', output_hidden_states = True, output_attentions = True) | |
| #model = GPT2LMHeadModel.from_pretrained('colorfulscoop/gpt2-small-ja',output_hidden_states= True, output_attentions=True) | |
| tokenizer = T5Tokenizer.from_pretrained("rinna/japanese-gpt-1b") | |
| model = AutoModelForCausalLM.from_pretrained("rinna/japanese-gpt-1b") | |
| def sentence_prob_mean(text): | |
| # Tokenize the input text and add special tokens | |
| input_ids = tokenizer.encode(text, return_tensors='pt') | |
| # Obtain model outputs | |
| with torch.no_grad(): | |
| outputs = model(input_ids, labels=input_ids) | |
| logits = outputs.logits # logits are the model outputs before applying softmax | |
| # Shift logits and labels so that tokens are aligned: | |
| shift_logits = logits[..., :-1, :].contiguous() | |
| shift_labels = input_ids[..., 1:].contiguous() | |
| # Calculate the softmax probabilities | |
| probs = softmax(shift_logits, dim=-1) | |
| # Gather the probabilities of the actual token IDs | |
| gathered_probs = torch.gather(probs, 2, shift_labels.unsqueeze(-1)).squeeze(-1) | |
| # Compute the mean probability across the tokens | |
| mean_prob = torch.mean(gathered_probs).item() | |
| return mean_prob | |
| #model = gr.Interface.load('huggingface/distilgpt2', output_hidden_states = True, output_attentions = True) | |
| #model.eval() | |
| #tokenizer = gr.Interface.load('huggingface/distilgpt2') | |
| #tokenizer = GPT2Tokenizer.from_pretrained('distilgpt2') | |
| #tokenizer = T5Tokenizer.from_pretrained('colorfulscoop/gpt2-small-ja') | |
| #tokenizer = GPT2Tokenizer.from_pretrained('distilgpt2') | |
| # def cloze_prob(text): | |
| # whole_text_encoding = tokenizer.encode(text) | |
| # # Parse out the stem of the whole sentence (i.e., the part leading up to but not including the critical word) | |
| # text_list = text.split() | |
| # stem = ' '.join(text_list[:-1]) | |
| # stem_encoding = tokenizer.encode(stem) | |
| # # cw_encoding is just the difference between whole_text_encoding and stem_encoding | |
| # # note: this might not correspond exactly to the word itself | |
| # cw_encoding = whole_text_encoding[len(stem_encoding):] | |
| # # Run the entire sentence through the model. Then go "back in time" to look at what the model predicted for each token, starting at the stem. | |
| # # Put the whole text encoding into a tensor, and get the model's comprehensive output | |
| # tokens_tensor = torch.tensor([whole_text_encoding]) | |
| # with torch.no_grad(): | |
| # outputs = model(tokens_tensor) | |
| # predictions = outputs[0] | |
| # logprobs = [] | |
| # # start at the stem and get downstream probabilities incrementally from the model(see above) | |
| # start = -1-len(cw_encoding) | |
| # for j in range(start,-1,1): | |
| # raw_output = [] | |
| # for i in predictions[-1][j]: | |
| # raw_output.append(i.item()) | |
| # logprobs.append(np.log(softmax(raw_output))) | |
| # # if the critical word is three tokens long, the raw_probabilities should look something like this: | |
| # # [ [0.412, 0.001, ... ] ,[0.213, 0.004, ...], [0.002,0.001, 0.93 ...]] | |
| # # Then for the i'th token we want to find its associated probability | |
| # # this is just: raw_probabilities[i][token_index] | |
| # conditional_probs = [] | |
| # for cw,prob in zip(cw_encoding,logprobs): | |
| # conditional_probs.append(prob[cw]) | |
| # # now that you have all the relevant probabilities, return their product. | |
| # # This is the probability of the critical word given the context before it. | |
| # return np.exp(np.sum(conditional_probs)) | |
| def cos_sim(a, b): | |
| return np.inner(a, b) / (np.linalg.norm(a) * (np.linalg.norm(b))) | |
| def get_sim(x): | |
| x = str(x)[1:-1] | |
| x = str(x)[1:-1] | |
| return x | |
| #def Visual_re_ranker(caption, visual_context_label, visual_context_prob): | |
| def Visual_re_ranker(sentence_man, sentence_woman, context_label, context_prob): | |
| sentence_man = sentence_man | |
| sentence_woman = sentence_woman | |
| context_label= context_label | |
| context_prob = context_prob | |
| sentence_emb_man = model_sbert.encode(sentence_man, convert_to_tensor=True) | |
| sentence_emb_woman = model_sbert.encode(sentence_woman, convert_to_tensor=True) | |
| context_label_emb = model_sbert.encode(context_label, convert_to_tensor=True) | |
| sim_m = cosine_scores = util.pytorch_cos_sim(sentence_emb_man, context_label_emb) | |
| sim_m = sim_m.cpu().numpy() | |
| sim_m = get_sim(sim_m) | |
| sim_w = cosine_scores = util.pytorch_cos_sim(sentence_emb_woman, context_label_emb) | |
| sim_w = sim_w.cpu().numpy() | |
| sim_w = get_sim(sim_w) | |
| LM_man = sentence_prob_mean(sentence_man) | |
| LM_woman = sentence_prob_mean(sentence_woman) | |
| #LM_man = cloze_prob(sentence_man) | |
| #LM_woman = cloze_prob(sentence_woman) | |
| score_man = pow(float(LM_man),pow((1-float(sim_m))/(1+ float(sim_m)),1-float(context_prob))) | |
| score_woman = pow(float(LM_woman),pow((1-float(sim_w))/(1+ float(sim_w)),1-float(context_prob))) | |
| #return {"彼 (man)": float(score_man * 100000000), "彼女 (woman)": float(score_woman)* 1000000000} | |
| return {"彼 (man)": float(score_man), "彼女 (woman)": float(score_woman)} | |
| #print(Visual_re_ranker("ハイデルベルク大学は彼の出身大学である。", "大学", "0.7458009")) | |
| demo = gr.Interface( | |
| fn=Visual_re_ranker, | |
| description="Demo for Women Wearing Lipstick: Measuring the Bias Between Object and Its Related Gender (JP) ", | |
| inputs=[gr.Textbox(value="ハイデルベルク大学は彼の出身大学である。") , gr.Textbox(value="ハイデルベルク大学は彼女の出身大学である。"), gr.Textbox(value="大学"), gr.Textbox(value="0.7458009")], | |
| # inputs=[gr.Textbox(value="これこれ！！なっちょのインスタ開設はこれがあるから尚幸せなのよ！") , gr.Textbox(value="インスタ開設"), gr.Textbox(value="大学"), gr.Textbox(value="0.5239")], | |
| #inputs=[gr.Textbox(value="a man is blow drying his hair in the bathroom") , gr.Textbox(value="a woman is blow drying her hair in the bathroom"), gr.Textbox(value="hair spray"), gr.Textbox(value="0.7385")], | |
| #outputs=[gr.Textbox(value="Language Model Score") , gr.Textbox(value="Semantic Similarity Score"), gr.Textbox(value="Belief revision score via visual context")], | |
| outputs="label", | |
| ) | |
| demo.launch() | |