Spaces:
Runtime error
Runtime error
File size: 3,269 Bytes
d6585f5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 |
/*
Copyright (c) 2008 - Chris Buckley.
Permission is granted for use and modification of this file for
research, non-commercial purposes.
*/
#include "common.h"
#include "sysfunc.h"
#include "trec_eval.h"
#include "functions.h"
#include "trec_format.h"
static int
te_calc_gm_bpref (const EPI *epi, const REL_INFO *rel_info,
const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval);
/* See trec_eval.h for definition of TREC_MEAS */
TREC_MEAS te_meas_gm_bpref =
{"gm_bpref",
" Binary preference (bpref), but using goemetric mean over topics\n\
See the explanation for 'bpref' for the base measure for a single topic.\n\
Gm_bpref uses the geometric mean to combine the single topic scores.\n\
This rewards methods that are more consistent across topics as opposed to\n\
high scores for some topics and low scores for others.\n\
Gm_bpref is printed only as a summary measure across topics, not for the\n\
individual topics.\n",
te_init_meas_s_float,
te_calc_gm_bpref,
te_acc_meas_s,
te_calc_avg_meas_s_gm,
te_print_single_meas_empty,
te_print_final_meas_s_float,
NULL, -1};
static int
te_calc_gm_bpref (const EPI *epi, const REL_INFO *rel_info,
const RESULTS *results, const TREC_MEAS *tm, TREC_EVAL *eval)
{
RES_RELS res_rels;
long j;
long nonrel_so_far, rel_so_far, pool_unjudged_so_far;
long num_nonrel = 0;
double bpref = 0.0;
if (UNDEF == te_form_res_rels (epi, rel_info, results, &res_rels))
return (UNDEF);
for (j = 0; j < epi->relevance_level; j++)
num_nonrel += res_rels.rel_levels[j];
/* Calculate judgement based measures (dependent on only
judged docs; no assumption of non-relevance if not judged) */
/* Binary Preference measures; here expressed as all docs with a higher
value of rel are to be preferred. Optimize by keeping track of nonrel
seen so far */
nonrel_so_far = 0;
rel_so_far = 0;
pool_unjudged_so_far = 0;
for (j = 0; j < res_rels.num_ret; j++) {
if (res_rels.results_rel_list[j] == RELVALUE_NONPOOL)
/* document not in pool. Skip */
continue;
if (res_rels.results_rel_list[j] == RELVALUE_UNJUDGED) {
/* document in pool but unjudged. */
pool_unjudged_so_far++;
continue;
}
if (res_rels.results_rel_list[j] >= 0 &&
res_rels.results_rel_list[j] < epi->relevance_level)
nonrel_so_far++;
else {
/* Judged Rel doc */
rel_so_far++;
/* Add fraction of correct preferences. */
/* Special case nonrel_so_far == 0 to avoid division by 0 */
if (nonrel_so_far > 0) {
bpref += 1.0 -
(((double) MIN (nonrel_so_far, res_rels.num_rel)) /
(double) MIN (num_nonrel, res_rels.num_rel));
}
else
bpref += 1.0;
}
}
if (res_rels.num_rel)
bpref /= res_rels.num_rel;
/* Original measure value is constrained to be greater than
MIN_GEO_MEAN (for time being .00001, since trec_eval prints to
four significant digits) */
eval->values[tm->eval_index].value =
(double) log ((double)(MAX (bpref, MIN_GEO_MEAN)));
if (epi->debug_level > 1)
printf ("gm_bpref: bpref %6.4f, gm_bpref %6.4f",
bpref, eval->values[tm->eval_index].value);
return (1);
}
|