Repository: hpcaitech/ColossalAI
File: applications/Chat/evaluate/evaluator.py (Python)

Code before:

            category_metrics = self.params[category]["Metrics"]
            self.automatic_metric_stats[category] = {}

            targets_list = [
                target["target"] if target["target"] else target["output"] for target in targets_per_category[category]
            ]
            predicts_list = [answer["output"] for answer in answers_per_category[category]]

            for metric in category_metrics:
                self.automatic_metric_stats[category].update(switch(metric=metric))

        # gpt evaluation
        for category in self.params:
            if len(answers_per_category[category]) == 0:
                print(f"Category {category} specified in your config doesn't have corresponding answers!")
                continue

            category_metrics = self.params[category]["GPT"]

            prompt = self.gpt_evaluation_prompt.get(category, None)
            if prompt is None:
                print(f"No prompt for category {category}! Use prompt for category general now.")
                prompt = self.gpt_evaluation_prompt["general"]

            self.gpt_evaluation_results[category] = gpt_evaluate.gpt_evaluate(answers_per_category[category], prompt,
                                                                              category_metrics, category,
                                                                              self.gpt_model)

    def save(self, path: str, model_name_list: List[str]) -> None:
        """
        Save evaluation results of GPT-3.5, GPT-4, and off-the-shelf evaluation metrics.

        """

        if len(model_name_list) == 2:
            save_path = os.path.join(path, "gpt_evaluate", "battle_results")
            gpt_evaluate.save_battle_results(self.battle_results, model_name_list[0], model_name_list[1], save_path)
        else:
            # save evaluation results for automatic metrics
            automatic_df = pd.DataFrame(self.automatic_metric_stats)

            automatic_results_save_path = os.path.join(path, "automatic_results")
            if not os.path.exists(automatic_results_save_path):
                os.makedirs(automatic_results_save_path)
            automatic_df.to_csv(os.path.join(automatic_results_save_path, f"{model_name_list[0]}.csv"), index=True)

            # Save evaluation results for GPT-3.5 evaluation metrics.
            all_evaluations = []
            base_save_path = os.path.join(path, "gpt_evaluate", "gpt_evaluate_results")
            evaluation_results_save_path = os.path.join(base_save_path, "evaluation_results")

Review comment (suggestion):
`gpt_evaluate.gpt_evaluate` is a bit confusing. Please rename it to `gpt_evaluate.evaluate` or other function names that shows the differences.

Code after:

            category_metrics = self.params[category]["Metrics"]
            self.automatic_metric_stats[category] = {}

            targets_list = [
                target["target"] if target["target"] else target["output"] for target in targets_per_category[category]
            ]
            predicts_list = [answer["output"] for answer in answers_per_category[category]]

            for metric in category_metrics:
                self.automatic_metric_stats[category].update(switch(metric=metric))

        # gpt evaluation
        for category in self.params:
            if len(answers_per_category[category]) == 0:
                print(f"Category {category} specified in your config doesn't have corresponding answers!")
                continue

            category_metrics = self.params[category]["GPT"]

            prompt = self.gpt_evaluation_prompt.get(category, None)
            if prompt is None:
                print(f"No prompt for category {category}! Use prompt for category general now.")
                prompt = self.gpt_evaluation_prompt["general"]

            self.gpt_evaluation_results[category] = gpt_evaluate.evaluate(answers_per_category[category], prompt,
                                                                          category_metrics, category, self.gpt_model)

    def save(self, path: str, model_name_list: List[str]) -> None:
        """
        Save evaluation results of GPT-3.5, GPT-4, and off-the-shelf evaluation metrics.

        """

        if len(model_name_list) == 2:
            save_path = os.path.join(path, "gpt_evaluate", "battle_results")
            gpt_evaluate.save_battle_results(self.battle_results, model_name_list[0], model_name_list[1], save_path)
        else:
            # save evaluation results for automatic metrics
            automatic_df = pd.DataFrame(self.automatic_metric_stats)

            automatic_results_save_path = os.path.join(path, "automatic_results")
            if not os.path.exists(automatic_results_save_path):
                os.makedirs(automatic_results_save_path)
            automatic_df.to_csv(os.path.join(automatic_results_save_path, f"{model_name_list[0]}.csv"), index=True)

            # Save evaluation results for GPT-3.5 evaluation metrics.
            all_evaluations = []
            base_save_path = os.path.join(path, "gpt_evaluate", "gpt_evaluate_results")
            evaluation_results_save_path = os.path.join(base_save_path, "evaluation_results")