Repository: hpcaitech/ColossalAI
File: applications/Chat/evaluate/eval.py (Python)

Code before:
import argparse
import json

from evaluator import Evaluator
from utils import jload

def main(args):
     # load config
    config=jload(args.config_file)

    if config["language"] == "cn":
        # get metric settings for all categories
        metrics_per_category = {}
        for category in config["category"].keys():
            metrics_all = {}
            for metric_type, metrics in config["category"][category].items():
                metrics_all[metric_type]=metrics
            metrics_per_category[category]=metrics_all

        answers = {}
        targets = {}

        with open(args.target_file, 'r', encoding='utf-8') as f1:
            targets = json.load(f1)

        with open(args.predict_file, 'r', encoding='utf-8') as f2:
            answers = json.load(f2)
            
        # initialize evaluator
        evaluator = Evaluator(metrics_per_category)
        evaluator.evaluate(answers = answers, targets = targets)
        evaluator.save(args.save_path)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--config_file', type=str, default=None, help='path to the file of target results')
    parser.add_argument('--target_file', type=str, default=None, help='path to the file of target results')
    parser.add_argument('--predict_file', type=str, default=None, help='path to the file of model prediction results')
    parser.add_argument('--save_path', type=str, default="results.csv", help='path to the csv file to save evaluation results')
    args = parser.parse_args()
    main(args)

Review comment (suggestion):
You config used to initialize  evaluator should consider whether you use `battle` or `evaluate`. You can control this in your config file or determine this based on your `predict_file`. If it has two files, then you should call `battle`, otherwise you should call `evaluate`.



Code after:
import argparse
import json

from evaluator import Evaluator
from utils import jload

def main(args):
     # load config
    config = jload(args.config_file)

    if config["language"] == "cn":
        # get metric settings for all categories
        metrics_per_category = {}
        for category in config["category"].keys():
            metrics_all = {}
            for metric_type, metrics in config["category"][category].items():
                metrics_all[metric_type] = metrics
            metrics_per_category[category] = metrics_all
            
        # initialize evaluator
        evaluator = Evaluator(metrics_per_category)
        if args.answers2_file:
            answers1 = jload(args.answers1_file)
            answers2 = jload(args.answers2_file)
            evaluator.battle(answers1=answers1, answers2=answers2)
        else:
            targets = jload(args.target_file)
            answers = jload(args.answers1_file)
            evaluator.evaluate(answers=answers, targets=targets)
            evaluator.save(args.save_path)
    else:
        raise ValueError(f'Unsupported language')


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--config_file', type=str, default=None, help='path to the file of target results')
    parser.add_argument('--target_file', type=str, default=None, help='path to the file of target results')
    parser.add_argument('--answers1_file', type=str, default=None, help='path to the file of one model prediction')
    parser.add_argument('--answers2_file', type=str, default=None, help='path to the file of the other model prediction')
    parser.add_argument('--save_path', type=str, default="results.csv", help='path to the csv file to save evaluation results')
    args = parser.parse_args()
    main(args)