Spaces:
Runtime error
Runtime error
File size: 2,954 Bytes
d6585f5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 |
/*
Copyright (c) 2008 - Chris Buckley.
Permission is granted for use and modification of this file for
research, non-commercial purposes.
*/
#include "common.h"
#include "sysfunc.h"
#include "trec_eval.h"
#include "functions.h"
#include "trec_format.h"
static int
te_calc_bpref (const EPI *epi, const REL_INFO *rel_info, const RESULTS *results,
const TREC_MEAS *tm, TREC_EVAL *eval);
/* See trec_eval.h for definition of TREC_MEAS */
TREC_MEAS te_meas_bpref =
{"bpref",
" Main binary preference measure.\n\
Fraction of the top R nonrelevant docs that are retrieved after each\n\
relevant doc. Put another way: when looking at the R relevant docs, and\n\
the top R nonrelevant docs, if all relevant docs are to be preferred to\n\
nonrelevant docs, bpref is the fraction of the preferences that the\n\
ranking preserves.\n\
Cite: 'Retrieval Evaluation with Incomplete Information', Chris Buckley\n\
and Ellen Voorhees. In Proceedings of 27th SIGIR, 2004.\n",
te_init_meas_s_float,
te_calc_bpref,
te_acc_meas_s,
te_calc_avg_meas_s,
te_print_single_meas_s_float,
te_print_final_meas_s_float,
NULL, -1};
static int
te_calc_bpref (const EPI *epi, const REL_INFO *rel_info, const RESULTS *results,
const TREC_MEAS *tm, TREC_EVAL *eval)
{
RES_RELS res_rels;
long j;
long nonrel_so_far, rel_so_far, pool_unjudged_so_far;
long num_nonrel = 0;
double bpref = 0.0;
if (UNDEF == te_form_res_rels (epi, rel_info, results, &res_rels))
return (UNDEF);
for (j = 0; j < epi->relevance_level; j++)
num_nonrel += res_rels.rel_levels[j];
/* Calculate judgement based measures (dependent on only
judged docs; no assumption of non-relevance if not judged) */
/* Binary Preference measures; here expressed as all docs with a higher
value of rel are to be preferred. Optimize by keeping track of nonrel
seen so far */
nonrel_so_far = 0;
rel_so_far = 0;
pool_unjudged_so_far = 0;
for (j = 0; j < res_rels.num_ret; j++) {
if (res_rels.results_rel_list[j] == RELVALUE_NONPOOL)
/* document not in pool. Skip */
continue;
if (res_rels.results_rel_list[j] == RELVALUE_UNJUDGED) {
/* document in pool but unjudged. */
pool_unjudged_so_far++;
continue;
}
if (res_rels.results_rel_list[j] >= 0 &&
res_rels.results_rel_list[j] < epi->relevance_level)
nonrel_so_far++;
else {
/* Judged Rel doc */
rel_so_far++;
/* Add fraction of correct preferences. */
/* Special case nonrel_so_far == 0 to avoid division by 0 */
if (nonrel_so_far > 0) {
bpref += 1.0 -
(((double) MIN (nonrel_so_far, res_rels.num_rel)) /
(double) MIN (num_nonrel, res_rels.num_rel));
}
else
bpref += 1.0;
}
}
if (res_rels.num_rel)
bpref /= res_rels.num_rel;
eval->values[tm->eval_index].value = bpref;
return (1);
}
|