Spaces:
Runtime error
Runtime error
/* | |
Copyright (c) 2008 - Chris Buckley. | |
Permission is granted for use and modification of this file for | |
research, non-commercial purposes. | |
*/ | |
static int | |
te_calc_bpref (const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, | |
const TREC_MEAS *tm, TREC_EVAL *eval); | |
/* See trec_eval.h for definition of TREC_MEAS */ | |
TREC_MEAS te_meas_bpref = | |
{"bpref", | |
" Main binary preference measure.\n\ | |
Fraction of the top R nonrelevant docs that are retrieved after each\n\ | |
relevant doc. Put another way: when looking at the R relevant docs, and\n\ | |
the top R nonrelevant docs, if all relevant docs are to be preferred to\n\ | |
nonrelevant docs, bpref is the fraction of the preferences that the\n\ | |
ranking preserves.\n\ | |
Cite: 'Retrieval Evaluation with Incomplete Information', Chris Buckley\n\ | |
and Ellen Voorhees. In Proceedings of 27th SIGIR, 2004.\n", | |
te_init_meas_s_float, | |
te_calc_bpref, | |
te_acc_meas_s, | |
te_calc_avg_meas_s, | |
te_print_single_meas_s_float, | |
te_print_final_meas_s_float, | |
NULL, -1}; | |
static int | |
te_calc_bpref (const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, | |
const TREC_MEAS *tm, TREC_EVAL *eval) | |
{ | |
RES_RELS res_rels; | |
long j; | |
long nonrel_so_far, rel_so_far, pool_unjudged_so_far; | |
long num_nonrel = 0; | |
double bpref = 0.0; | |
if (UNDEF == te_form_res_rels (epi, rel_info, results, &res_rels)) | |
return (UNDEF); | |
for (j = 0; j < epi->relevance_level; j++) | |
num_nonrel += res_rels.rel_levels[j]; | |
/* Calculate judgement based measures (dependent on only | |
judged docs; no assumption of non-relevance if not judged) */ | |
/* Binary Preference measures; here expressed as all docs with a higher | |
value of rel are to be preferred. Optimize by keeping track of nonrel | |
seen so far */ | |
nonrel_so_far = 0; | |
rel_so_far = 0; | |
pool_unjudged_so_far = 0; | |
for (j = 0; j < res_rels.num_ret; j++) { | |
if (res_rels.results_rel_list[j] == RELVALUE_NONPOOL) | |
/* document not in pool. Skip */ | |
continue; | |
if (res_rels.results_rel_list[j] == RELVALUE_UNJUDGED) { | |
/* document in pool but unjudged. */ | |
pool_unjudged_so_far++; | |
continue; | |
} | |
if (res_rels.results_rel_list[j] >= 0 && | |
res_rels.results_rel_list[j] < epi->relevance_level) | |
nonrel_so_far++; | |
else { | |
/* Judged Rel doc */ | |
rel_so_far++; | |
/* Add fraction of correct preferences. */ | |
/* Special case nonrel_so_far == 0 to avoid division by 0 */ | |
if (nonrel_so_far > 0) { | |
bpref += 1.0 - | |
(((double) MIN (nonrel_so_far, res_rels.num_rel)) / | |
(double) MIN (num_nonrel, res_rels.num_rel)); | |
} | |
else | |
bpref += 1.0; | |
} | |
} | |
if (res_rels.num_rel) | |
bpref /= res_rels.num_rel; | |
eval->values[tm->eval_index].value = bpref; | |
return (1); | |
} | |