File size: 4,655 Bytes
8c6b5ee |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 |
"""
Goal
---
1. Read test results from log.txt files
2. Compute mean and std across different folders (seeds)
Usage
---
Assume the output files are saved under output/my_experiment,
which contains results of different seeds, e.g.,
my_experiment/
seed1/
log.txt
seed2/
log.txt
seed3/
log.txt
Run the following command from the root directory:
$ python tools/parse_test_res.py output/my_experiment
Add --ci95 to the argument if you wanna get 95% confidence
interval instead of standard deviation:
$ python tools/parse_test_res.py output/my_experiment --ci95
If my_experiment/ has the following structure,
my_experiment/
exp-1/
seed1/
log.txt
...
seed2/
log.txt
...
seed3/
log.txt
...
exp-2/
...
exp-3/
...
Run
$ python tools/parse_test_res.py output/my_experiment --multi-exp
"""
import re
import numpy as np
import os.path as osp
import argparse
from collections import OrderedDict, defaultdict
from dassl.utils import check_isfile, listdir_nohidden
def compute_ci95(res):
return 1.96 * np.std(res) / np.sqrt(len(res))
def parse_function(*metrics, directory="", args=None, end_signal=None):
print("===")
print(f"Parsing files in {directory}")
subdirs = listdir_nohidden(directory, sort=True)
outputs = []
for subdir in subdirs:
fpath = osp.join(directory, subdir, "log.txt")
assert check_isfile(fpath)
good_to_go = False
output = OrderedDict()
with open(fpath, "r") as f:
lines = f.readlines()
for line in lines:
line = line.strip()
if line == end_signal:
good_to_go = True
for metric in metrics:
match = metric["regex"].search(line)
if match and good_to_go:
if "file" not in output:
output["file"] = fpath
num = float(match.group(1))
name = metric["name"]
output[name] = num
if output:
outputs.append(output)
assert len(outputs) > 0, f"Nothing found in {directory}"
metrics_results = defaultdict(list)
for output in outputs:
msg = ""
for key, value in output.items():
if isinstance(value, float):
msg += f"{key}: {value:.1f}%. "
else:
msg += f"{key}: {value}. "
if key != "file":
metrics_results[key].append(value)
print(msg)
output_results = OrderedDict()
for key, values in metrics_results.items():
avg = np.mean(values)
std = compute_ci95(values) if args.ci95 else np.std(values)
print(f"* average {key}: {avg:.1f}% +- {std:.1f}%")
output_results[key] = avg
print("===")
return output_results
def main(args, end_signal):
metric = {
"name": args.keyword,
"regex": re.compile(fr"\* {args.keyword}: ([\.\deE+-]+)%"),
}
if args.multi_exp:
final_results = defaultdict(list)
for directory in listdir_nohidden(args.directory, sort=True):
directory = osp.join(args.directory, directory)
results = parse_function(
metric, directory=directory, args=args, end_signal=end_signal
)
for key, value in results.items():
final_results[key].append(value)
print("Average performance")
for key, values in final_results.items():
avg = np.mean(values)
print(f"* {key}: {avg:.1f}%")
else:
parse_function(
metric, directory=args.directory, args=args, end_signal=end_signal
)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("directory", type=str, help="path to directory")
parser.add_argument(
"--ci95",
action="store_true",
help=r"compute 95\% confidence interval"
)
parser.add_argument(
"--test-log", action="store_true", help="parse test-only logs"
)
parser.add_argument(
"--multi-exp", action="store_true", help="parse multiple experiments"
)
parser.add_argument(
"--keyword",
default="accuracy",
type=str,
help="which keyword to extract"
)
args = parser.parse_args()
end_signal = "Finish training" # needs to be adapted to the latest
if args.test_log:
end_signal = "=> result"
main(args, end_signal)
|