constants.py

ALL_METRICS = [
    "ndcg_cut_10",
    "ndcg_cut_5",
    "ndcg_cut_15",
    "ndcg_cut_20", 
    "ndcg_cut_30", 
    "ndcg_cut_100", 
    "ndcg_cut_200", 
    "ndcg_cut_500", 
    "ndcg_cut_1000", 
    "map", 
    "P_5", 
    "P_10", 
    "P_15", 
    "P_20", 
    "P_30", 
    "P_100", 
    "P_200", 
    "P_500", 
    "P_1000", 
    "recall_5", 
    "recall_10", 
    "recall_15", 
    "recall_20", 
    "recall_30", 
    "recall_100", 
    "recall_200", 
    "recall_500", 
    "recall_1000", 
    "recip_rank", 
    "set_recall", 
    "set_P", 
    "set_F", 
    "num_rel_ret", 
    "num_ret", 
    "num_rel", 
    "num_q", 
    "num_rel", 
    "num_rel_ret"
    "Rprec",
    "bpref",
    "iprec_at_recall_0.00",
    "iprec_at_recall_0.10",
    "iprec_at_recall_0.20",
    "iprec_at_recall_0.30",
    "iprec_at_recall_0.40",
    "iprec_at_recall_0.50",
    "iprec_at_recall_0.60",
    "iprec_at_recall_0.70",
    "iprec_at_recall_0.80",
    "iprec_at_recall_0.90",
    "iprec_at_recall_1.00",
]


BEIR = [
    "msmarco",
    "trec-covid",
    "nf_corpus",
    "bioasq",
    "nq",
    "hotpotqa",
    "fiqa",
    "signal1m",
    "trec-news",
    "robust04",
    "arguana",
    "webis-touche2020",
    "cqadupstack",
    "quora",
    "dbpedia-entity",
    "scidocs",
    "fever",
    "climate-fever",
    "scifact",
]


IR_DATASETS = [
    "antique",
    "aol_ia",
    "aquaint",
    "argsme",
    "c4",
    "car",
    "clinicaltrials",
    "clirmatrix",
    "clueweb09",
    "clueweb12",
    "codec",
    "cord19",
    "cranfield",
    "disks45",
    "dpr_w100",
    "codesearchnet",
    "gov",
    "gov2",
    "highwire",
    "istella22",
    "kilt",
    "lotte",
    "medline",
    "mmarco",
    "mr_tydi",
    "msmarco_document",
    "msmarco_document_v2",
    "msmarco_passage",
    "msmarco_passage_v2",
    "msmarco_qna",
    "neumarco",
    "nfcorpus",
    "natural_questions",
    "nyt",
    "pmc",
    "touche_image",
    "touche",
    "trec_arabic",
    "trec_mandarin",
    "trec_spanish",
    "trec_robust04",
    "trec_tot",
    "tripclick",
    "tweets2013_ia",
    "vaswani",
    "wapo",
    "wikiclir",
    "wikir",
    "trec_fair",
    "trec_cast",
    "hc4",
    "neuclir",
    "sara",
]

LOCAL_DATASETS = [
    "gooaq_technical",
    "codesearch_py",
]
    
    
ALL_DATASETS = ["", "custom"] + LOCAL_DATASETS + BEIR + IR_DATASETS