Spaces:
Sleeping
Sleeping
| from __future__ import absolute_import | |
| from __future__ import division | |
| from __future__ import print_function | |
| import numpy as np | |
| import time | |
| from collections import OrderedDict | |
| import torch | |
| import sys | |
| try: | |
| sys.path.append("cider") | |
| from pyciderevalcap.ciderD.ciderD import CiderD | |
| from pyciderevalcap.cider.cider import Cider | |
| sys.path.append("coco-caption") | |
| from pycocoevalcap.bleu.bleu import Bleu | |
| except: | |
| print('cider or coco-caption missing') | |
| CiderD_scorer = None | |
| Cider_scorer = None | |
| Bleu_scorer = None | |
| #CiderD_scorer = CiderD(df='corpus') | |
| def init_scorer(cached_tokens): | |
| global CiderD_scorer | |
| CiderD_scorer = CiderD_scorer or CiderD(df=cached_tokens) | |
| global Cider_scorer | |
| Cider_scorer = Cider_scorer or Cider(df=cached_tokens) | |
| global Bleu_scorer | |
| Bleu_scorer = Bleu_scorer or Bleu(4) | |
| def array_to_str(arr): | |
| out = '' | |
| for i in range(len(arr)): | |
| out += str(arr[i]) + ' ' | |
| if arr[i] == 0: | |
| break | |
| return out.strip() | |
| def get_self_critical_reward(greedy_res, data_gts, gen_result, opt): | |
| batch_size = len(data_gts) | |
| gen_result_size = gen_result.shape[0] | |
| seq_per_img = gen_result_size // len(data_gts) # gen_result_size = batch_size * seq_per_img | |
| assert greedy_res.shape[0] == batch_size | |
| res = OrderedDict() | |
| gen_result = gen_result.data.cpu().numpy() | |
| greedy_res = greedy_res.data.cpu().numpy() | |
| for i in range(gen_result_size): | |
| res[i] = [array_to_str(gen_result[i])] | |
| for i in range(batch_size): | |
| res[gen_result_size + i] = [array_to_str(greedy_res[i])] | |
| gts = OrderedDict() | |
| for i in range(len(data_gts)): | |
| gts[i] = [array_to_str(data_gts[i][j]) for j in range(len(data_gts[i]))] | |
| res_ = [{'image_id':i, 'caption': res[i]} for i in range(len(res))] | |
| res__ = {i: res[i] for i in range(len(res_))} | |
| gts_ = {i: gts[i // seq_per_img] for i in range(gen_result_size)} | |
| gts_.update({i+gen_result_size: gts[i] for i in range(batch_size)}) | |
| if opt.cider_reward_weight > 0: | |
| _, cider_scores = CiderD_scorer.compute_score(gts_, res_) | |
| print('Cider scores:', _) | |
| else: | |
| cider_scores = 0 | |
| if opt.bleu_reward_weight > 0: | |
| _, bleu_scores = Bleu_scorer.compute_score(gts_, res__) | |
| bleu_scores = np.array(bleu_scores[3]) | |
| print('Bleu scores:', _[3]) | |
| else: | |
| bleu_scores = 0 | |
| scores = opt.cider_reward_weight * cider_scores + opt.bleu_reward_weight * bleu_scores | |
| scores = scores[:gen_result_size].reshape(batch_size, seq_per_img) - scores[-batch_size:][:, np.newaxis] | |
| scores = scores.reshape(gen_result_size) | |
| rewards = np.repeat(scores[:, np.newaxis], gen_result.shape[1], 1) | |
| return rewards | |
| def get_scores(data_gts, gen_result, opt): | |
| batch_size = gen_result.size(0)# batch_size = sample_size * seq_per_img | |
| seq_per_img = batch_size // len(data_gts) | |
| res = OrderedDict() | |
| gen_result = gen_result.data.cpu().numpy() | |
| for i in range(batch_size): | |
| res[i] = [array_to_str(gen_result[i])] | |
| gts = OrderedDict() | |
| for i in range(len(data_gts)): | |
| gts[i] = [array_to_str(data_gts[i][j]) for j in range(len(data_gts[i]))] | |
| res_ = [{'image_id':i, 'caption': res[i]} for i in range(batch_size)] | |
| res__ = {i: res[i] for i in range(batch_size)} | |
| gts = {i: gts[i // seq_per_img] for i in range(batch_size)} | |
| if opt.cider_reward_weight > 0: | |
| _, cider_scores = CiderD_scorer.compute_score(gts, res_) | |
| print('Cider scores:', _) | |
| else: | |
| cider_scores = 0 | |
| if opt.bleu_reward_weight > 0: | |
| _, bleu_scores = Bleu_scorer.compute_score(gts, res__) | |
| bleu_scores = np.array(bleu_scores[3]) | |
| print('Bleu scores:', _[3]) | |
| else: | |
| bleu_scores = 0 | |
| scores = opt.cider_reward_weight * cider_scores + opt.bleu_reward_weight * bleu_scores | |
| return scores | |
| def get_self_cider_scores(data_gts, gen_result, opt): | |
| batch_size = gen_result.size(0)# batch_size = sample_size * seq_per_img | |
| seq_per_img = batch_size // len(data_gts) | |
| res = [] | |
| gen_result = gen_result.data.cpu().numpy() | |
| for i in range(batch_size): | |
| res.append(array_to_str(gen_result[i])) | |
| scores = [] | |
| for i in range(len(data_gts)): | |
| tmp = Cider_scorer.my_self_cider([res[i*seq_per_img:(i+1)*seq_per_img]]) | |
| def get_div(eigvals): | |
| eigvals = np.clip(eigvals, 0, None) | |
| return -np.log(np.sqrt(eigvals[-1]) / (np.sqrt(eigvals).sum())) / np.log(len(eigvals)) | |
| scores.append(get_div(np.linalg.eigvalsh(tmp[0]/10))) | |
| scores = np.array(scores) | |
| return scores |