Spaces:
Sleeping
Sleeping
File size: 4,535 Bytes
be5548b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 |
import os
import sys
import argparse
import shutil
from collections import defaultdict
import numpy as np
import tensorflow as tf
from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
def tabulate_events(exp_path):
seeds = [s for s in os.listdir(exp_path) if "combined" not in s]
summary_iterators = [EventAccumulator(os.path.join(exp_path, dname)).Reload() for dname in seeds]
tags = summary_iterators[0].Tags()['scalars']
for it in summary_iterators:
assert it.Tags()['scalars'] == tags
out = defaultdict(list)
for tag in tags:
for events in zip(*[acc.Scalars(tag) for acc in summary_iterators]):
assert len(set(e.step for e in events)) == 1
out[tag].append([e.value for e in events])
return out
def create_histogram_summary(tag, values, bins=1000):
# Convert to a numpy array
values = np.array(values)
# Create histogram using numpy
counts, bin_edges = np.histogram(values, bins=bins)
# Fill fields of histogram proto
hist = tf.HistogramProto()
hist.min = float(np.min(values))
hist.max = float(np.max(values))
hist.num = int(np.prod(values.shape))
hist.sum = float(np.sum(values))
hist.sum_squares = float(np.sum(values**2))
# Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1]
# See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto#L30
# Thus, we drop the start of the first bin
bin_edges = bin_edges[1:]
# Add bin edges and counts
for edge in bin_edges:
hist.bucket_limit.append(edge)
for c in counts:
hist.bucket.append(c)
# Create and write Summary
return tf.Summary.Value(tag=tag, histo=hist)
def create_parsed_histogram_summary(tag, values, bins=1000):
# Convert to a numpy array
# Create histogram using numpy
counts, bin_edges = np.histogram(values, bins=bins)
# Fill fields of histogram proto
hist = tf.HistogramProto()
hist.min = float(np.min(values))
hist.max = float(np.max(values))
hist.num = int(np.prod(values.shape))
hist.sum = float(np.sum(values))
hist.sum_squares = float(np.sum(values**2))
# Requires equal number as bins, where the first goes from -DBL_MAX to bin_edges[1]
# See https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/summary.proto#L30
# Thus, we drop the start of the first bin
bin_edges = bin_edges[1:]
# Add bin edges and counts
for edge in bin_edges:
hist.bucket_limit.append(edge)
for c in counts:
hist.bucket.append(c)
# Create and write Summary
return tf.Summary.Value(tag=tag, histo=hist)
def write_combined_events(exp_path, d_combined, dname='combined', mean_var_tags=()):
fpath = os.path.join(exp_path, dname)
if os.path.isdir(fpath):
shutil.rmtree(fpath)
assert not os.path.isdir(fpath)
writer = tf.summary.FileWriter(fpath)
tags, values = zip(*d_combined.items())
cap = min([len(v) for v in values])
values = [v[:cap] for v in values]
timestep_mean = np.array(values).mean(axis=-1)
timestep_var = np.array(values).var(axis=-1)
timesteps = timestep_mean[tags.index("frames")]
for tag, means, vars in zip(tags, timestep_mean, timestep_var):
for i, mean, var in zip(timesteps, means, vars):
summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=mean)])
writer.add_summary(summary, global_step=i)
writer.flush()
if tag in mean_var_tags:
values = np.array([mean - var, mean, mean + var])
summary = tf.Summary(value=[
create_histogram_summary(tag=tag+"_var", values=values)
])
writer.add_summary(summary, global_step=i)
writer.flush()
if __name__ == "__main__":
if len(sys.argv) > 1:
dpath = sys.argv[1]
else:
raise ValueError("Specify dir")
parser = argparse.ArgumentParser()
parser.add_argument('--experiments', nargs='+', help='experiment directories to aggregate', required=True)
parser.add_argument('--mean-var-tags', nargs='+', help='tags to create mean-var histograms from', required=False, default=["return_mean"])
args = parser.parse_args()
for exp_path in args.experiments:
d = tabulate_events(exp_path)
write_combined_events(exp_path, d, mean_var_tags=args.mean_var_tags) |