Repository: beir-cellar/beir
Issue: Incorrect Evaluation Metrics Produced
Root cause: Documentation Deficiency

`def beir_evaluation():

    actual_contexts_dict = {'0': 
                               {'0': 1, '1': 1, '2': 1, '3': 1, '4': 1, '5': 1, '6': 1, '7': 1, '8': 1, '9': 1}
                            }
    results_dict = {'0': 
                       {'0': 1, '1': 1, '2': 1, '3': 1, '4': 1, '5': 1, '6': 1, '7': 1, '8': 1, '9': 1}
                    }

    # Evaluate retrieval metrics
    ndcg, map_score, recall, precision = EvaluateRetrieval.evaluate(
        actual_contexts_dict, results_dict, k_values=[10]
    )
    mrr = EvaluateRetrieval.evaluate_custom(actual_contexts_dict, results_dict, [10], metric="mrr")

    # Print evaluation results
    print(f"\nRecall@10   : {recall['Recall@10']:.2f}")
    print(f"Precision@10: {precision['P@10']:.2f}")
    print(f"\nNDCG@10     : {ndcg['NDCG@10']:.2f}")
    print(f"MAP@10      : {map_score['MAP@10']:.2f}")
    print(f"MRR@10      : {mrr['MRR@10']:.2f}")`

Recall@10   : 0.90
Precision@10: 0.90

NDCG@10     : 0.94
MAP@10      : 0.90
MRR@10      : 1.00


This is clearly incorrect as everything should be 1. Please can this be resolved


```
from beir.retrieval.evaluation import EvaluateRetrieval

def beir_evaluation():

    actual_contexts_dict = {'0':{'0': 0, '1': 1, '2': 2, '3': 3, '4': 4, '5': 5, '6': 6, '7': 7, '8': 8, '9': 9},
                            '1':{'10': 10, '11': 11, '12': 12, '13': 13, '14': 14, '15': 15, '16': 16, '17': 17, '18': 18, '19': 19}
                            }
    results_dict = {'0':{'0': 0, '1': 0, '2': 0, '3': 0, '4': 0, '5': 0, '6': 0, '7': 0, '8': 0, '9': 0},
                    '1':{'10': 10, '11': 11, '12': 12, '13': 13, '14': 14, '15': 15, '16': 16, '17': 17, '18': 18, '19': 19}
                    }

    # Evaluate retrieval metrics
    ndcg, map_score, recall, precision = EvaluateRetrieval.evaluate(
        actual_contexts_dict, results_dict, k_values=[10]
    )
    mrr = EvaluateRetrieval.evaluate_custom(actual_contexts_dict, results_dict, [10], metric="mrr")

    # Print evaluation results
    print(f"\nRecall@10   : {recall['Recall@10']:.2f}")
    print(f"Precision@10: {precision['P@10']:.2f}")
    print(f"\nNDCG@10     : {ndcg['NDCG@10']:.2f}")
    print(f"MAP@10      : {map_score['MAP@10']:.2f}")
    print(f"MRR@10      : {mrr['MRR@10']:.2f}")

# Run the evaluation
beir_evaluation()
```


Recall@10   : 1.00
Precision@10: 0.95

NDCG@10     : 1.00
MAP@10      : 1.00
MRR@10      : 1.00

Why is precision@10 not 1.00 ?

> `def beir_evaluation():
> 
> ```
> actual_contexts_dict = {'0': 
>                            {'0': 1, '1': 1, '2': 1, '3': 1, '4': 1, '5': 1, '6': 1, '7': 1, '8': 1, '9': 1}
>                         }
> results_dict = {'0': 
>                    {'0': 1, '1': 1, '2': 1, '3': 1, '4': 1, '5': 1, '6': 1, '7': 1, '8': 1, '9': 1}
>                 }
> 
> # Evaluate retrieval metrics
> ndcg, map_score, recall, precision = EvaluateRetrieval.evaluate(
>     actual_contexts_dict, results_dict, k_values=[10],  ignore_identical_ids=False
> )    # please make sure to add the ignore_identical_ids parameters to False to consider all the documents or use the different Query and Documents_ID
> mrr = EvaluateRetrieval.evaluate_custom(actual_contexts_dict, results_dict, [10], metric="mrr")
> 
> # Print evaluation results
> print(f"\nRecall@10   : {recall['Recall@10']:.2f}")
> print(f"Precision@10: {precision['P@10']:.2f}")
> print(f"\nNDCG@10     : {ndcg['NDCG@10']:.2f}")
> print(f"MAP@10      : {map_score['MAP@10']:.2f}")
> print(f"MRR@10      : {mrr['MRR@10']:.2f}")`
> ```
> #### please make sure to add the ignore_identical_ids parameters to False to consider all the documents or use the different Query and Documents_ID
> #please go through this code for better understanding:
https://github.com/beir-cellar/beir/blob/main/beir/retrieval/evaluation.py



BEIR use pytrec_eval to evaluate. Maybe the solutions can be found in pytrec_eval issuses. 

```python
y = [[1,2,3,4,5,6,7,8,9,10],
     [11,12,13,14,15,16,17,18,19,20],
     [21,22,23,24,25,26,27,28,29,30]
     ]
y_hat = [[8, 1, 4, 9, 7, 3, 5, 6, 10, 2],
         [15, 13, 18, 11, 20, 14, 17, 16, 19, 12],
         [27, 28, 23, 21, 29, 30, 22, 25, 26, 24]
        ]

evaluator = Beir_Evaluator(y)
evaluator.evaluate_retrieval(y_hat)
```
<b>Output</b>
Recall@10   : 1.00
Precision@10: 1.00
NDCG@10     : 1.00
MAP@10      : 1.00
MRR@10      : 1.00

```python
from beir.retrieval.evaluation import EvaluateRetrieval

class Beir_Evaluator():

    def __init__(self,benchmarks):
        self.benchmarks = benchmarks

    def _get_beir_format(self,results):

        return {
            str(i): {str(item): int(item) for item in sublist}
            for i, sublist in enumerate(results)
        }

    def evaluate_retrieval(self,retrieved_contexts,k_vals = [10]):
        
        actual_contexts = self._get_beir_format(self.benchmarks)
        retrieved_contexts = self._get_beir_format(retrieved_contexts)

        ndcg, map_score, recall, precision = EvaluateRetrieval.evaluate(
        actual_contexts, retrieved_contexts, k_values=k_vals
        )
        mrr = EvaluateRetrieval.evaluate_custom(actual_contexts, retrieved_contexts, k_vals, metric="mrr")

        # Print evaluation results
        for k in k_vals:
            print(f"\n{k}:")
            print(f"Recall@{k}   : {recall[f'Recall@{k}']:.2f}")
            print(f"Precision@{k}: {precision[f'P@{k}']:.2f}")
            print(f"NDCG@{k}     : {ndcg[f'NDCG@{k}']:.2f}")
            print(f"MAP@{k}      : {map_score[f'MAP@{k}']:.2f}")
            print(f"MRR@{k}      : {mrr[f'MRR@{k}']:.2f}")
```

Would this be a good approach and is there any benefit in creating a PR ?

The metrics produced in BEIR are correct. 
One should be careful with query_id and document_id as shown in the reply @shivareddy0117 (thanks for that!).

> y = [[1,2,3,4,5,6,7,8,9,10],
>      [11,12,13,14,15,16,17,18,19,20],
>      [21,22,23,24,25,26,27,28,29,30]
>      ]
> y_hat = [[8, 1, 4, 9, 7, 3, 5, 6, 10, 2],
>          [15, 13, 18, 11, 20, 14, 17, 16, 19, 12],
>          [27, 28, 23, 21, 29, 30, 22, 25, 26, 24]
>         ]
> 
> evaluator = Beir_Evaluator(y)
> evaluator.evaluate_retrieval(y_hat)
> Output Recall@10 : 1.00 Precision@10: 1.00 NDCG@10 : 1.00 MAP@10 : 1.00 MRR@10 : 1.00
> 
> from beir.retrieval.evaluation import EvaluateRetrieval
> 
> class Beir_Evaluator():
> 
>     def __init__(self,benchmarks):
>         self.benchmarks = benchmarks
> 
>     def _get_beir_format(self,results):
> 
>         return {
>             str(i): {str(item): int(item) for item in sublist}
>             for i, sublist in enumerate(results)
>         }
> 
>     def evaluate_retrieval(self,retrieved_contexts,k_vals = [10]):
>         
>         actual_contexts = self._get_beir_format(self.benchmarks)
>         retrieved_contexts = self._get_beir_format(retrieved_contexts)
> 
>         ndcg, map_score, recall, precision = EvaluateRetrieval.evaluate(
>         actual_contexts, retrieved_contexts, k_values=k_vals
>         )
>         mrr = EvaluateRetrieval.evaluate_custom(actual_contexts, retrieved_contexts, k_vals, metric="mrr")
> 
>         # Print evaluation results
>         for k in k_vals:
>             print(f"\n{k}:")
>             print(f"Recall@{k}   : {recall[f'Recall@{k}']:.2f}")
>             print(f"Precision@{k}: {precision[f'P@{k}']:.2f}")
>             print(f"NDCG@{k}     : {ndcg[f'NDCG@{k}']:.2f}")
>             print(f"MAP@{k}      : {map_score[f'MAP@{k}']:.2f}")
>             print(f"MRR@{k}      : {mrr[f'MRR@{k}']:.2f}")
> Would this be a good approach and is there any benefit in creating a PR?

@NikhielRahulSingh This code is very specific, I can extend the current evaluation to make it easier to use without retrieval etc.