Spaces:
Build error
Build error
File size: 2,589 Bytes
32408ed |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 |
"""Helper functions for computing evaluation scores"""
import numpy as np
from cirtorch.utils.evaluate import compute_map
def compute_map_and_log(dataset, ranks, gnd, kappas=(1, 5, 10), logger=None):
"""Computed mAP and log it
:param str dataset: Dataset to compute the mAP on (e.g. roxford5k)
:param np.ndarray ranks: 2D matrix of ints corresponding to previously computed ranks
:param dict gnd: Ground-truth dataset structure
:param list kappas: Compute mean precision at each kappa
:param logging.Logger logger: If not None, use it to log mAP and all mP@kappa
:return tuple: mAP and mP@kappa (medium difficulty for roxford5k and rparis6k)
"""
# new evaluation protocol
if dataset.startswith('roxford5k') or dataset.startswith('rparis6k'):
gnd_t = []
for gndi in gnd:
g = {}
g['ok'] = np.concatenate([gndi['easy']])
g['junk'] = np.concatenate([gndi['junk'], gndi['hard']])
gnd_t.append(g)
mapE, apsE, mprE, prsE = compute_map(ranks, gnd_t, kappas)
gnd_t = []
for gndi in gnd:
g = {}
g['ok'] = np.concatenate([gndi['easy'], gndi['hard']])
g['junk'] = np.concatenate([gndi['junk']])
gnd_t.append(g)
mapM, apsM, mprM, prsM = compute_map(ranks, gnd_t, kappas)
gnd_t = []
for gndi in gnd:
g = {}
g['ok'] = np.concatenate([gndi['hard']])
g['junk'] = np.concatenate([gndi['junk'], gndi['easy']])
gnd_t.append(g)
mapH, apsH, mprH, prsH = compute_map(ranks, gnd_t, kappas)
if logger:
fmap = lambda x: np.around(x*100, decimals=2)
logger.info(f"Evaluated {dataset}: mAP E: {fmap(mapE)}, M: {fmap(mapM)}, H: {fmap(mapH)}")
logger.info(f"Evaluated {dataset}: mP@k{kappas} E: {fmap(mprE)}, M: {fmap(mprM)}, H: {fmap(mprH)}")
scores = {"map_easy": mapE.item(), "mp@k_easy": mprE, "ap_easy": apsE, "p@k_easy": prsE,
"map_medium": mapM.item(), "mp@k_medium": mprM, "ap_medium": apsM, "p@k_medium": prsM,
"map_hard": mapH.item(), "mp@k_hard": mprH, "ap_hard": apsH, "p@k_hard": prsH}
return scores
# old evaluation protocol
map_score, ap_scores, prk, pr_scores = compute_map(ranks, gnd, kappas=kappas)
if logger:
fmap = lambda x: np.around(x*100, decimals=2)
logger.info(f"Evaluated {dataset}: mAP {fmap(map_score)}, mP@k {fmap(prk)}")
return {"map": map_score, "mp@k": prk, "ap": ap_scores, "p@k": pr_scores}
|