Repository: hpcaitech/ColossalAI
File: applications/Chat/evaluate/evaluator.py (Python)

Code before:
            for dict in targets:
                if dict["category"] == category:
                    if(dict["target"]):
                        targets_list.append(dict["target"])
                    else:
                        targets_list.append(dict["output"])

            for dict in answers: 
                if dict["category"] == category:
                    predicts_list.append(dict["output"])

            for metric in category_metrics:
                if (metric == "BLEU"):
                    self.stats[category].update(metrics.bleu_score(preds=predicts_list, targets=targets_list))
                elif (metric == "ROUGE"):
                    self.stats[category].update(metrics.rouge_cn_score(preds=predicts_list, targets=targets_list))
                elif (metric == "Distinct"):
                    self.stats[category].update(metrics.distinct_score(preds=predicts_list))             
                elif (metric == "BERTScore"):
                    self.stats[category].update(metrics.bert_score(preds=predicts_list, targets=targets_list))       
                elif (metric == "Precision"): 
                    self.stats[category].update(metrics.precision(preds=predicts_list, targets=targets_list))
                elif (metric == "Recall"):
                    self.stats[category].update(metrics.recall(preds=predicts_list, targets=targets_list))
                else:
                    self.stats[category].update(metrics.F1_score(preds=predicts_list, targets=targets_list))

    def save(self, path: str) -> None:
        # automatic evaluation result
        automatic_df = pd.DataFrame(self.stats)
        automatic_df.to_csv(path,index=True)


Review comment (suggestion):
These can be designed as a dict which maps the `metric` to target function to avoid `if else` statement.

Code after:
            elif (metric == "Distinct"):
                return metrics.distinct_score(preds=predicts_list)
            elif (metric == "BERTScore"):
                return metrics.bert_score(preds=predicts_list, targets=targets_list)
            elif (metric == "Precision"):
                return metrics.precision(preds=predicts_list, targets=targets_list)
            elif (metric == "Recall"):
                return metrics.recall(preds=predicts_list, targets=targets_list)
            elif (metric == "F1 score"):
                return metrics.F1_score(preds=predicts_list, targets=targets_list)
            else:
                raise ValueError(f"Unexpected metric")

        # automatic evaluation
        for category in self.params:
            category_metrics = self.params[category]["Metrics"]
            targets_list = []
            predicts_list = []
            self.stats[category] = {}
            
            for dict in targets:
                if dict["category"] == category:
                    if(dict["target"]):
                        targets_list.append(dict["target"])
                    else:
                        targets_list.append(dict["output"])

            for dict in answers: 
                if dict["category"] == category:
                    predicts_list.append(dict["output"])

            for metric in category_metrics:
                self.stats[category].update(switch(metric=metric))

    def save(self, path: str) -> None:
        # automatic evaluation result
        automatic_df = pd.DataFrame(self.stats)
        automatic_df.to_csv(path,index=True)