Spaces:
Running
Running
| import os | |
| import sys | |
| import argparse | |
| import shutil | |
| from collections import defaultdict | |
| import numpy as np | |
| import tensorflow as tf | |
| from tensorboard.backend.event_processing.event_accumulator import EventAccumulator | |
| def tabulate_events(exp_path): | |
| seeds = [s for s in os.listdir(exp_path) if "combined" not in s] | |
| summary_iterators = [EventAccumulator(os.path.join(exp_path, dname)).Reload() for dname in seeds] | |
| tags = summary_iterators[0].Tags()['scalars'] | |
| for it in summary_iterators: | |
| assert it.Tags()['scalars'] == tags | |
| out = defaultdict(list) | |
| for tag in tags: | |
| for events in zip(*[acc.Scalars(tag) for acc in summary_iterators]): | |
| assert len(set(e.step for e in events)) == 1 | |
| out[tag].append([e.value for e in events]) | |
| return out | |
| def create_histogram_summary(tag, values, bins=1000): | |
| # Convert to a numpy array | |
| values = np.array(values) | |
| # Create histogram using numpy | |
| counts, bin_edges = np.histogram(values, bins=bins) | |
| # Fill fields of histogram proto | |
| hist = tf.HistogramProto() | |
| hist.min = float(np.min(values)) | |
| hist.max = float(np.max(values)) | |
| hist.num = int(np.prod(values.shape)) | |
| hist.sum = float(np.sum(values)) | |
| hist.sum_squares = float(np.sum(values**2)) | |
| # Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1] | |
| # See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto#L30 | |
| # Thus, we drop the start of the first bin | |
| bin_edges = bin_edges[1:] | |
| # Add bin edges and counts | |
| for edge in bin_edges: | |
| hist.bucket_limit.append(edge) | |
| for c in counts: | |
| hist.bucket.append(c) | |
| # Create and write Summary | |
| return tf.Summary.Value(tag=tag, histo=hist) | |
| def create_parsed_histogram_summary(tag, values, bins=1000): | |
| # Convert to a numpy array | |
| # Create histogram using numpy | |
| counts, bin_edges = np.histogram(values, bins=bins) | |
| # Fill fields of histogram proto | |
| hist = tf.HistogramProto() | |
| hist.min = float(np.min(values)) | |
| hist.max = float(np.max(values)) | |
| hist.num = int(np.prod(values.shape)) | |
| hist.sum = float(np.sum(values)) | |
| hist.sum_squares = float(np.sum(values**2)) | |
| # Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1] | |
| # See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto#L30 | |
| # Thus, we drop the start of the first bin | |
| bin_edges = bin_edges[1:] | |
| # Add bin edges and counts | |
| for edge in bin_edges: | |
| hist.bucket_limit.append(edge) | |
| for c in counts: | |
| hist.bucket.append(c) | |
| # Create and write Summary | |
| return tf.Summary.Value(tag=tag, histo=hist) | |
| def write_combined_events(exp_path, d_combined, dname='combined', mean_var_tags=()): | |
| fpath = os.path.join(exp_path, dname) | |
| if os.path.isdir(fpath): | |
| shutil.rmtree(fpath) | |
| assert not os.path.isdir(fpath) | |
| writer = tf.summary.FileWriter(fpath) | |
| tags, values = zip(*d_combined.items()) | |
| cap = min([len(v) for v in values]) | |
| values = [v[:cap] for v in values] | |
| timestep_mean = np.array(values).mean(axis=-1) | |
| timestep_var = np.array(values).var(axis=-1) | |
| timesteps = timestep_mean[tags.index("frames")] | |
| for tag, means, vars in zip(tags, timestep_mean, timestep_var): | |
| for i, mean, var in zip(timesteps, means, vars): | |
| summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=mean)]) | |
| writer.add_summary(summary, global_step=i) | |
| writer.flush() | |
| if tag in mean_var_tags: | |
| values = np.array([mean - var, mean, mean + var]) | |
| summary = tf.Summary(value=[ | |
| create_histogram_summary(tag=tag+"_var", values=values) | |
| ]) | |
| writer.add_summary(summary, global_step=i) | |
| writer.flush() | |
| if __name__ == "__main__": | |
| if len(sys.argv) > 1: | |
| dpath = sys.argv[1] | |
| else: | |
| raise ValueError("Specify dir") | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument('--experiments', nargs='+', help='experiment directories to aggregate', required=True) | |
| parser.add_argument('--mean-var-tags', nargs='+', help='tags to create mean-var histograms from', required=False, default=["return_mean"]) | |
| args = parser.parse_args() | |
| for exp_path in args.experiments: | |
| d = tabulate_events(exp_path) | |
| write_combined_events(exp_path, d, mean_var_tags=args.mean_var_tags) |