Spaces:
Runtime error
Runtime error
| /* | |
| Copyright (c) 2008 - Chris Buckley. | |
| Permission is granted for use and modification of this file for | |
| research, non-commercial purposes. | |
| */ | |
| static int | |
| te_calc_bpref (const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, | |
| const TREC_MEAS *tm, TREC_EVAL *eval); | |
| /* See trec_eval.h for definition of TREC_MEAS */ | |
| TREC_MEAS te_meas_bpref = | |
| {"bpref", | |
| " Main binary preference measure.\n\ | |
| Fraction of the top R nonrelevant docs that are retrieved after each\n\ | |
| relevant doc. Put another way: when looking at the R relevant docs, and\n\ | |
| the top R nonrelevant docs, if all relevant docs are to be preferred to\n\ | |
| nonrelevant docs, bpref is the fraction of the preferences that the\n\ | |
| ranking preserves.\n\ | |
| Cite: 'Retrieval Evaluation with Incomplete Information', Chris Buckley\n\ | |
| and Ellen Voorhees. In Proceedings of 27th SIGIR, 2004.\n", | |
| te_init_meas_s_float, | |
| te_calc_bpref, | |
| te_acc_meas_s, | |
| te_calc_avg_meas_s, | |
| te_print_single_meas_s_float, | |
| te_print_final_meas_s_float, | |
| NULL, -1}; | |
| static int | |
| te_calc_bpref (const EPI *epi, const REL_INFO *rel_info, const RESULTS *results, | |
| const TREC_MEAS *tm, TREC_EVAL *eval) | |
| { | |
| RES_RELS res_rels; | |
| long j; | |
| long nonrel_so_far, rel_so_far, pool_unjudged_so_far; | |
| long num_nonrel = 0; | |
| double bpref = 0.0; | |
| if (UNDEF == te_form_res_rels (epi, rel_info, results, &res_rels)) | |
| return (UNDEF); | |
| for (j = 0; j < epi->relevance_level; j++) | |
| num_nonrel += res_rels.rel_levels[j]; | |
| /* Calculate judgement based measures (dependent on only | |
| judged docs; no assumption of non-relevance if not judged) */ | |
| /* Binary Preference measures; here expressed as all docs with a higher | |
| value of rel are to be preferred. Optimize by keeping track of nonrel | |
| seen so far */ | |
| nonrel_so_far = 0; | |
| rel_so_far = 0; | |
| pool_unjudged_so_far = 0; | |
| for (j = 0; j < res_rels.num_ret; j++) { | |
| if (res_rels.results_rel_list[j] == RELVALUE_NONPOOL) | |
| /* document not in pool. Skip */ | |
| continue; | |
| if (res_rels.results_rel_list[j] == RELVALUE_UNJUDGED) { | |
| /* document in pool but unjudged. */ | |
| pool_unjudged_so_far++; | |
| continue; | |
| } | |
| if (res_rels.results_rel_list[j] >= 0 && | |
| res_rels.results_rel_list[j] < epi->relevance_level) | |
| nonrel_so_far++; | |
| else { | |
| /* Judged Rel doc */ | |
| rel_so_far++; | |
| /* Add fraction of correct preferences. */ | |
| /* Special case nonrel_so_far == 0 to avoid division by 0 */ | |
| if (nonrel_so_far > 0) { | |
| bpref += 1.0 - | |
| (((double) MIN (nonrel_so_far, res_rels.num_rel)) / | |
| (double) MIN (num_nonrel, res_rels.num_rel)); | |
| } | |
| else | |
| bpref += 1.0; | |
| } | |
| } | |
| if (res_rels.num_rel) | |
| bpref /= res_rels.num_rel; | |
| eval->values[tm->eval_index].value = bpref; | |
| return (1); | |
| } | |