Spaces:
Sleeping
Sleeping
File size: 5,953 Bytes
03f6091 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 |
import json
from polos.metrics.regression_metrics import RegressionReport
from polos.models import load_checkpoint
from tqdm import tqdm
import json
from polos.models import download_model, load_checkpoint, model2download, str2model
from polos.trainer import TrainerConfig, build_trainer
import yaml
from utils import *
from dataset import *
from pascal50s import Pascal50sDataset
from PIL import Image
from pathlib import Path
from copy import deepcopy
class FoilDatset:
def __init__(self, coco_root_path="data_en/coco", foil_path="data_en/foil/foilv1.0_test_2017.json"):
coco_root_path = Path(coco_root_path)
coco_path = coco_root_path / Path("captions_val2014.json")
coco_refs = self._read_coco(coco_path)
self.data = self._build_foil(foil_path, coco_refs) # data[anno_id][foil or orig] = [anno1, anno2, ...]
self.coco_root_path = coco_root_path
self.dataset = {"one_ref" : None, "four_ref" : None}
def _read_coco(self, coco_annos):
refs = {}
with open(coco_annos) as f:
coco = json.load(f)
for ann in coco["annotations"]:
refs.setdefault(ann['image_id'],[]).append(ann['caption'])
return refs
def _build_foil(self, path, coco_refs):
with open(path) as f:
self.data = json.load(f)
images = self.data["images"]
annos = self.data["annotations"]
data = {}
imgid_to_img = {img["id"] : img for img in images}
for anno in annos:
anno_id = anno["id"]
data.setdefault(anno_id, {"foil" : [], "orig" : []})
key = "foil" if anno["foil"] else "orig"
anno["image"] = imgid_to_img[anno["image_id"]]
anno["refs"] = coco_refs[anno["image_id"]]
data[anno_id][key].append(anno)
return data
def get_data(self,one_ref):
key = "one_ref" if one_ref else "four_ref"
if self.dataset[key] is not None:
return self.dataset[key]
dataset = []
for _, data in (pbar := tqdm(self.data.items())): # data[anno_id][foil or orig] = [anno1, anno2, ...]
pbar.set_description("Prepare dataset ...")
foiles, origs = data["foil"], data["orig"]
assert len(origs) == 1
N = len(foiles)
for foil, orig in zip(foiles, [origs[0]]*N):
refs = foil["refs"]
refs = [r for r in refs if r != orig["caption"]]
if one_ref:
refs = [refs[0]]
filename = Path(foil["image"]["file_name"])
img_path = Path("data_en/images") / filename
dataset.append({
"imgid" : img_path,
"refs": refs,
"mt": foil["caption"],
"type": "foil"
})
dataset.append({
"imgid" : img_path,
"refs": refs,
"mt": orig["caption"],
"type": "orig"
})
self.dataset[key] = dataset
return self.dataset[key]
def collect_acc(memory, dataset_name, method, acc):
memory.setdefault(dataset_name, {})
memory[dataset_name].update({method : acc})
gprint(f"[{dataset_name}]",method,acc)
def polos(dataset,args):
yprint("Compute Polos ...")
rep = RegressionReport()
if args.model:
model = load_checkpoint(args.model)
elif args.hparams:
yaml_file = yaml.load(open(args.hparams).read(), Loader=yaml.FullLoader)
train_configs = TrainerConfig(yaml_file)
model_config = str2model[train_configs.model].ModelConfig(yaml_file)
print(str2model[train_configs.model].ModelConfig)
print(model_config.namespace())
model = str2model[train_configs.model](model_config.namespace())
model.eval()
model.freeze()
data = []
gt_scores = []
for data_ in (pbar := tqdm(dataset)):
pbar.set_description("Prepare dataset ...")
data.append(data_)
_, sys_score = model.predict(data,cuda=True,batch_size=32)
return sys_score
def compute_acc(model_fn,dataset,one_ref,**kwargs):
# Split by buckets because images do not fit in RAM.
bucket_count = 10
data = dataset.get_data(one_ref)
print("Compute ...")
sys_score = []
for i in range(bucket_count):
bucket_size = len(data) // bucket_count
subset = deepcopy(data[i*bucket_size:(i+1)*bucket_size])
for j, sub in enumerate(pbar := tqdm(subset)):
pbar.set_description(f"Processing {i+1}/{bucket_count}")
subset[j].update({"img" : Image.open(sub["imgid"]).convert("RGB")})
sub_sys_score = model_fn(subset,**kwargs)
sys_score.extend(sub_sys_score)
del subset
assert len(sys_score) == len(data)
assert len(sys_score) % 2 == 0
acc = 0.
N = len(sys_score) // 2
for i in range(0,2*N,2):
s1 = sys_score[i] # foil
s2 = sys_score[i+1] # orig
# sanity check
assert data[i]["type"] == "foil" and data[i+1]["type"] == "orig"
if s2 > s1:
acc += 1.
acc /= N
rprint(f"acc: {acc}")
return acc
def compute_foil(args, memory, tops):
dataset = FoilDatset()
dataset_name = "foil"
for one_ref in [True, False]:
suffix = "(one_ref)" if one_ref else "(four-ref)"
dataset_name += suffix
if args.polos:
polos_acc = compute_acc(polos, dataset, one_ref, args=args)
collect_acc(memory, dataset_name, f"Polos{suffix}", polos_acc)
# aggregate
max_acc = ("", 0.)
for method, acc in memory[dataset_name].items():
if max_acc[1] < acc:
max_acc = (method, acc)
rprint("[TOP]")
rprint(max_acc)
tops[dataset_name] = max_acc
return memory, tops |