def __init__(self, time_series, baseline_time_series=None, smoothing_factor=None, use_lag_window=False, lag_window_size=None):

super(ExpAvgDetector, self).__init__(self.__class__.__name__, time_series, baseline_time_series)<EOL>self.use_lag_window = use_lag_window<EOL>self.smoothing_factor = smoothing_factor if smoothing_factor > <NUM_LIT:0> else DEFAULT_EMA_SMOOTHING_FACTOR<EOL>self.lag_window_size = lag_window_size if lag_window_size else int(self.time_series_length * DEFAULT_EMA_WINDOW_SIZE_PCT)<EOL>self.time_series_items = self.time_series.items()<EOL>

Initializer
:param TimeSeries time_series: a TimeSeries object.
:param TimeSeries baseline_time_series: baseline TimeSeries.
:param float smoothing_factor: smoothing factor for computing exponential moving average.
:param int lag_window_size: lagging window size.


def _compute_anom_score(self, lag_window_points, point):

ema = utils.compute_ema(self.smoothing_factor, lag_window_points)[-<NUM_LIT:1>]<EOL>return abs(point - ema)<EOL>

Compute anomaly score for a single data point.
Anomaly score for a single data point(t,v) equals: abs(v - ema(lagging window)).
:param list lag_window_points: values in the lagging window.
:param float point: data point value.
:return float: the anomaly score.


anom_scores = {}<EOL>values = self.time_series.values<EOL>stdev = numpy.std(values)<EOL>for i, (timestamp, value) in enumerate(self.time_series_items):<EOL><INDENT>if i < self.lag_window_size:<EOL><INDENT>anom_score = self._compute_anom_score(values[:i + <NUM_LIT:1>], value)<EOL><DEDENT>else:<EOL><INDENT>anom_score = self._compute_anom_score(values[i - self.lag_window_size: i + <NUM_LIT:1>], value)<EOL><DEDENT>if stdev:<EOL><INDENT>anom_scores[timestamp] = anom_score / stdev<EOL><DEDENT>else:<EOL><INDENT>anom_scores[timestamp] = anom_score<EOL><DEDENT><DEDENT>self.anom_scores = TimeSeries(self._denoise_scores(anom_scores))<EOL>

Compute anomaly scores using a lagging window.


anom_scores = {}<EOL>values = self.time_series.values<EOL>ema = utils.compute_ema(self.smoothing_factor, values)<EOL>stdev = numpy.std(values)<EOL>for i, (timestamp, value) in enumerate(self.time_series_items):<EOL><INDENT>anom_score = abs((value - ema[i]) / stdev) if stdev else value - ema[i]<EOL>anom_scores[timestamp] = anom_score<EOL><DEDENT>self.anom_scores = TimeSeries(self._denoise_scores(anom_scores))<EOL>

Compute anomaly scores using a lagging window covering all the data points before.


if self.use_lag_window:<EOL><INDENT>self._compute_anom_data_using_window()<EOL><DEDENT>self._compute_anom_data_decay_all()<EOL>

Compute anomaly scores for the time series.
Currently uses a lagging window covering all the data points before.


def __init__(self, class_name, time_series, baseline_time_series=None):

self.class_name = class_name<EOL>self.time_series = time_series<EOL>self.time_series_length = len(time_series)<EOL>self.baseline_time_series = baseline_time_series<EOL>

Initializer
:param str class_name: extended class name.
:param TimeSeries time_series: a TimeSeries object.
:param TimeSeries baseline_time_series: baseline TimeSeries.


self._set_scores()<EOL>return self.anom_scores<EOL>

Run the algorithm to get anomalies.
return list: a list of Anomaly objects.


if scores:<EOL><INDENT>maximal = max(scores.values())<EOL>if maximal:<EOL><INDENT>for key in scores:<EOL><INDENT>if scores[key] < DEFAULT_NOISE_PCT_THRESHOLD * maximal:<EOL><INDENT>scores[key] = <NUM_LIT:0><EOL><DEDENT><DEDENT><DEDENT><DEDENT>return scores<EOL>

Denoise anomaly scores.
Low anomaly scores could be noisy. The following two series will have good correlation result with out denoise:
[0.08, 4.6, 4.6, 4.6, 1.0, 1.0]
[0.0010, 0.0012, 0.0012, 0.0008, 0.0008]
while the second series is pretty flat(suppose it has a max score of 100).
param dict scores: the scores to be denoised.


raise NotImplementedError<EOL>

Compute anomaly scores for the time series.


return self.anom_scores<EOL>

Get anomaly scores for the time series.
:return TimeSeries: a TimeSeries representation of the anomaly scores.


def __init__(self, time_series, baseline_time_series=None, precision=None, lag_window_size=None,<EOL>future_window_size=None, chunk_size=None):

super(BitmapDetector, self).__init__(self.__class__.__name__, time_series, baseline_time_series)<EOL>self.precision = precision if precision and precision > <NUM_LIT:0> else DEFAULT_BITMAP_PRECISION<EOL>self.chunk_size = chunk_size if chunk_size and chunk_size > <NUM_LIT:0> else DEFAULT_BITMAP_CHUNK_SIZE<EOL>if lag_window_size:<EOL><INDENT>self.lag_window_size = lag_window_size<EOL><DEDENT>else:<EOL><INDENT>self.lag_window_size = int(self.time_series_length * DEFAULT_BITMAP_LAGGING_WINDOW_SIZE_PCT)<EOL><DEDENT>if future_window_size:<EOL><INDENT>self.future_window_size = future_window_size<EOL><DEDENT>else:<EOL><INDENT>self.future_window_size = int(self.time_series_length * DEFAULT_BITMAP_LEADING_WINDOW_SIZE_PCT)<EOL><DEDENT>self._sanity_check()<EOL>

Initializer
:param TimeSeries time_series: a TimeSeries object.
:param TimeSeries baseline_time_series: baseline TimeSeries.
:param int precision: how many sections to categorize values.
:param int lag_window_size: lagging window size.
:param int future_window_size: future window size.
:param int chunk_size: chunk size.


windows = self.lag_window_size + self.future_window_size<EOL>if (not self.lag_window_size or not self.future_window_size<EOL>or self.time_series_length < windows or windows < DEFAULT_BITMAP_MINIMAL_POINTS_IN_WINDOWS):<EOL><INDENT>raise exceptions.NotEnoughDataPoints<EOL><DEDENT>if self.lag_window_size > DEFAULT_BITMAP_MAXIMAL_POINTS_IN_WINDOWS:<EOL><INDENT>self.lag_window_size = DEFAULT_BITMAP_MAXIMAL_POINTS_IN_WINDOWS<EOL><DEDENT>if self.future_window_size > DEFAULT_BITMAP_MAXIMAL_POINTS_IN_WINDOWS:<EOL><INDENT>self.future_window_size = DEFAULT_BITMAP_MAXIMAL_POINTS_IN_WINDOWS<EOL><DEDENT>


def _generate_SAX_single(self, sections, value):

sax = <NUM_LIT:0><EOL>for section_number in sections.keys():<EOL><INDENT>section_lower_bound = sections[section_number]<EOL>if value >= section_lower_bound:<EOL><INDENT>sax = section_number<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>return str(sax)<EOL>

Generate SAX representation(Symbolic Aggregate approXimation) for a single data point.
Read more about it here: Assumption-Free Anomaly Detection in Time Series(http://alumni.cs.ucr.edu/~ratana/SSDBM05.pdf).
:param dict sections: value sections.
:param float value: value to be categorized.
:return str: a SAX representation.


sections = {}<EOL>self.value_min = self.time_series.min()<EOL>self.value_max = self.time_series.max()<EOL>section_height = (self.value_max - self.value_min) / self.precision<EOL>for section_number in range(self.precision):<EOL><INDENT>sections[section_number] = self.value_min + section_number * section_height<EOL><DEDENT>self.sax = '<STR_LIT>'.join(self._generate_SAX_single(sections, value) for value in self.time_series.values)<EOL>

Generate SAX representation for all values of the time series.


frequency = defaultdict(int)<EOL>chunk_size = self.chunk_size<EOL>length = len(sax)<EOL>for i in range(length):<EOL><INDENT>if i + chunk_size <= length:<EOL><INDENT>chunk = sax[i: i + chunk_size]<EOL>frequency[chunk] += <NUM_LIT:1><EOL><DEDENT><DEDENT>return frequency<EOL>

Form a chunk frequency dictionary from a SAX representation.
:param str sax: a SAX representation.
:return dict: frequency dictionary for chunks in the SAX representation.


lag_dicts = {}<EOL>fut_dicts = {}<EOL>length = self.time_series_length<EOL>lws = self.lag_window_size<EOL>fws = self.future_window_size<EOL>chunk_size = self.chunk_size<EOL>for i in range(length):<EOL><INDENT>if i < lws or i > length - fws:<EOL><INDENT>lag_dicts[i] = None<EOL><DEDENT>else:<EOL><INDENT>if lag_dicts[i - <NUM_LIT:1>] is None:<EOL><INDENT>lag_dict = self._construct_SAX_chunk_dict(self.sax[i - lws: i])<EOL>lag_dicts[i] = lag_dict<EOL>lw_leave_chunk = self.sax[<NUM_LIT:0>:chunk_size]<EOL>lw_enter_chunk = self.sax[i - chunk_size + <NUM_LIT:1>: i + <NUM_LIT:1>]<EOL>fut_dict = self._construct_SAX_chunk_dict(self.sax[i: i + fws])<EOL>fut_dicts[i] = fut_dict<EOL>fw_leave_chunk = self.sax[i: i + chunk_size]<EOL>fw_enter_chunk = self.sax[i + fws + <NUM_LIT:1> - chunk_size: i + fws + <NUM_LIT:1>]<EOL><DEDENT>else:<EOL><INDENT>lag_dict = copy(lag_dicts[i - <NUM_LIT:1>])<EOL>lag_dict[lw_leave_chunk] -= <NUM_LIT:1><EOL>lag_dict[lw_enter_chunk] += <NUM_LIT:1><EOL>lag_dicts[i] = lag_dict<EOL>fut_dict = copy(fut_dicts[i - <NUM_LIT:1>])<EOL>fut_dict[fw_leave_chunk] -= <NUM_LIT:1><EOL>fut_dict[fw_enter_chunk] += <NUM_LIT:1><EOL>fut_dicts[i] = fut_dict<EOL>lw_leave_chunk = self.sax[i - lws: i - lws + chunk_size]<EOL>lw_enter_chunk = self.sax[i - chunk_size + <NUM_LIT:1>: i + <NUM_LIT:1>]<EOL>fw_leave_chunk = self.sax[i: i + chunk_size]<EOL>fw_enter_chunk = self.sax[i + fws + <NUM_LIT:1> - chunk_size: i + fws + <NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT>self.lag_dicts = lag_dicts<EOL>self.fut_dicts = fut_dicts<EOL>

Construct the chunk dicts for lagging window and future window at each index.
 e.g: Suppose we have a SAX sequence as '1234567890', both window sizes are 3, and the chunk size is 2.
 The first index that has a lagging window is 3. For index equals 3, the lagging window has sequence '123',
 the chunk to leave lagging window(lw_leave_chunk) is '12', and the chunk to enter lagging window(lw_enter_chunk) is '34'.
 Therefore, given chunk dicts at i, to compute chunk dicts at i+1, simply decrement the count for lw_leave_chunk,
 and increment the count for lw_enter_chunk from chunk dicts at i. Same method applies to future window as well.


lag_window_chunk_dict = self.lag_dicts[i]<EOL>future_window_chunk_dict = self.fut_dicts[i]<EOL>score = <NUM_LIT:0><EOL>for chunk in lag_window_chunk_dict:<EOL><INDENT>if chunk in future_window_chunk_dict:<EOL><INDENT>score += math.pow(future_window_chunk_dict[chunk] - lag_window_chunk_dict[chunk], <NUM_LIT:2>)<EOL><DEDENT>else:<EOL><INDENT>score += math.pow(lag_window_chunk_dict[chunk], <NUM_LIT:2>)<EOL><DEDENT><DEDENT>for chunk in future_window_chunk_dict:<EOL><INDENT>if chunk not in lag_window_chunk_dict:<EOL><INDENT>score += math.pow(future_window_chunk_dict[chunk], <NUM_LIT:2>)<EOL><DEDENT><DEDENT>return score<EOL>

Compute distance difference between two windows' chunk frequencies,
which is then marked as the anomaly score of the data point on the window boundary in the middle.
:param int i: index of the data point between two windows.
:return float: the anomaly score.


anom_scores = {}<EOL>self._generate_SAX()<EOL>self._construct_all_SAX_chunk_dict()<EOL>length = self.time_series_length<EOL>lws = self.lag_window_size<EOL>fws = self.future_window_size<EOL>for i, timestamp in enumerate(self.time_series.timestamps):<EOL><INDENT>if i < lws or i > length - fws:<EOL><INDENT>anom_scores[timestamp] = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>anom_scores[timestamp] = self._compute_anom_score_between_two_windows(i)<EOL><DEDENT><DEDENT>self.anom_scores = TimeSeries(self._denoise_scores(anom_scores))<EOL>

Compute anomaly scores for the time series by sliding both lagging window and future window.


def __init__(self, time_series, absolute_threshold_value_upper=None, absolute_threshold_value_lower=None,<EOL>baseline_time_series=None):

super(AbsoluteThreshold, self).__init__(self.__class__.__name__, time_series, baseline_time_series)<EOL>self.absolute_threshold_value_upper = absolute_threshold_value_upper<EOL>self.absolute_threshold_value_lower = absolute_threshold_value_lower<EOL>if not self.absolute_threshold_value_lower and not self.absolute_threshold_value_upper:<EOL><INDENT>raise exceptions.RequiredParametersNotPassed('<STR_LIT>'<EOL>'<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>

Initialize algorithm, check all required args are present

:param time_series: The current time series dict to run anomaly detection on
:param absolute_threshold_value_upper: Time series values above this are considered anomalies
:param absolute_threshold_value_lower: Time series values below this are considered anomalies
:param baseline_time_series: A no-op for now
:return:


anom_scores = {}<EOL>for timestamp, value in self.time_series.items():<EOL><INDENT>anom_scores[timestamp] = <NUM_LIT:0.0><EOL>if self.absolute_threshold_value_upper and value > self.absolute_threshold_value_upper:<EOL><INDENT>anom_scores[timestamp] = value - self.absolute_threshold_value_upper<EOL><DEDENT>if self.absolute_threshold_value_lower and value < self.absolute_threshold_value_lower:<EOL><INDENT>anom_scores[timestamp] = self.absolute_threshold_value_lower - value<EOL><DEDENT><DEDENT>self.anom_scores = TimeSeries(self._denoise_scores(anom_scores))<EOL>

Compute anomaly scores for the time series
This algorithm just takes the diff of threshold with current value as anomaly score


def __init__(self, time_series_a, time_series_b, max_shift_seconds=None, shift_impact=None):

super(CrossCorrelator, self).__init__(self.__class__.__name__, time_series_a, time_series_b)<EOL>self.shift_impact = shift_impact or DEFAULT_SHIFT_IMPACT<EOL>if max_shift_seconds is not None:<EOL><INDENT>self.max_shift_milliseconds = max_shift_seconds<EOL><DEDENT>else:<EOL><INDENT>self.max_shift_milliseconds = DEFAULT_ALLOWED_SHIFT_SECONDS * <NUM_LIT:1000><EOL><DEDENT>

Initializer
:param TimeSeries time_series_a: TimeSeries a.
:param TimeSeries time_series_b: TimeSeries b.
:param int max_shift_milliseconds: allowed maximal shift seconds.
:param time_period: if given, correlate the data inside the time period only.


correlations = []<EOL>shifted_correlations = []<EOL>self.time_series_a.normalize()<EOL>self.time_series_b.normalize()<EOL>a, b = self.time_series_a.align(self.time_series_b)<EOL>a_values, b_values = a.values, b.values<EOL>a_avg, b_avg = a.average(), b.average()<EOL>a_stdev, b_stdev = a.stdev(), b.stdev()<EOL>n = len(a)<EOL>denom = a_stdev * b_stdev * n<EOL>allowed_shift_step = self._find_allowed_shift(a.timestamps)<EOL>if allowed_shift_step:<EOL><INDENT>shift_upper_bound = allowed_shift_step<EOL>shift_lower_bound = -allowed_shift_step<EOL><DEDENT>else:<EOL><INDENT>shift_upper_bound = <NUM_LIT:1><EOL>shift_lower_bound = <NUM_LIT:0><EOL><DEDENT>for delay in range(shift_lower_bound, shift_upper_bound):<EOL><INDENT>delay_in_seconds = a.timestamps[abs(delay)] - a.timestamps[<NUM_LIT:0>]<EOL>if delay < <NUM_LIT:0>:<EOL><INDENT>delay_in_seconds = -delay_in_seconds<EOL><DEDENT>s = <NUM_LIT:0><EOL>for i in range(n):<EOL><INDENT>j = i + delay<EOL>if j < <NUM_LIT:0> or j >= n:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>s += ((a_values[i] - a_avg) * (b_values[j] - b_avg))<EOL><DEDENT><DEDENT>r = s / denom if denom != <NUM_LIT:0> else s<EOL>correlations.append([delay_in_seconds, r])<EOL>if self.max_shift_milliseconds:<EOL><INDENT>shifted_correlations.append(r * (<NUM_LIT:1> + float(delay_in_seconds) / self.max_shift_milliseconds * self.shift_impact))<EOL><DEDENT>else:<EOL><INDENT>shifted_correlations.append(r)<EOL><DEDENT><DEDENT>max_correlation = list(max(correlations, key=lambda k: k[<NUM_LIT:1>]))<EOL>max_shifted_correlation = max(shifted_correlations)<EOL>max_correlation.append(max_shifted_correlation)<EOL>self.correlation_result = CorrelationResult(*max_correlation)<EOL>

Detect correlation by computing correlation coefficients for all allowed shift steps,
then take the maximum.


init_ts = timestamps[<NUM_LIT:0>]<EOL>residual_timestamps = map(lambda ts: ts - init_ts, timestamps)<EOL>n = len(residual_timestamps)<EOL>return self._find_first_bigger(residual_timestamps, self.max_shift_milliseconds, <NUM_LIT:0>, n)<EOL>

Find the maximum allowed shift steps based on max_shift_milliseconds.
param list timestamps: timestamps of a time series.


def _find_first_bigger(self, timestamps, target, lower_bound, upper_bound):

while lower_bound < upper_bound:<EOL><INDENT>pos = lower_bound + (upper_bound - lower_bound) / <NUM_LIT:2><EOL>if timestamps[pos] > target:<EOL><INDENT>upper_bound = pos<EOL><DEDENT>else:<EOL><INDENT>lower_bound = pos + <NUM_LIT:1><EOL><DEDENT><DEDENT>return pos<EOL>

Find the first element in timestamps whose value is bigger than target.
param list values: list of timestamps(epoch number).
param target: target value.
param lower_bound: lower bound for binary search.
param upper_bound: upper bound for binary search.


def __init__(self, class_name, time_series_a, time_series_b):

self.class_name = class_name<EOL>self.time_series_a = time_series_a<EOL>self.time_series_b = time_series_b<EOL>

Initializer
:param class_name: name of extended class.
:param TimeSeries time_series_a: TimeSeries a.
:param TimeSeries time_series_b: TimeSeries b.


raise NotImplementedError<EOL>


return self.correlation_result<EOL>

Get correlation result.
:return CorrelationResult: a CorrelationResult object represents the correlation result.


self._detect_correlation()<EOL>return self.correlation_result<EOL>

Execute algorithm.
:return CorrelationResult: a CorrelationResult object represents the correlation result.


def __init__(self, anomalies, correlations):

self.anomalies = anomalies<EOL>self.correlations = correlations<EOL>self._analyze_root_causes()<EOL>

:param list anomalies: a list of `Anomaly` objects.
                       `Anomaly` is defined in luminol.modules.anomaly.

:param dict correlations: a dict represents correlated metrics(`TimeSeries` object) to each anomaly.
                          each key-value pair looks like this:
                          `Anomaly` --> [metric1, metric2, metric3 ...].


causes = {}<EOL>for a in self.anomalies:<EOL><INDENT>try:<EOL><INDENT>causes[a] = self.correlations[a][<NUM_LIT:0>]<EOL><DEDENT>except IndexError:<EOL><INDENT>raise exceptions.InvalidDataFormat('<STR_LIT>')<EOL><DEDENT><DEDENT>self.causes = causes<EOL>

Conduct root cause analysis.
The first metric of the list is taken as the root cause right now.


return getattr(self, '<STR_LIT>', None)<EOL>

Get root causes.
:return dict: a dict represents root causes for each anomaly.


ema = []<EOL>if(len(points) > <NUM_LIT:0>):<EOL><INDENT>ema.append(points[<NUM_LIT:0>])<EOL><DEDENT>for i in range(<NUM_LIT:1>, len(points)):<EOL><INDENT>ema.append(smoothing_factor * points[i] + (<NUM_LIT:1> - smoothing_factor) * ema[i - <NUM_LIT:1>])<EOL><DEDENT>return ema<EOL>

Compute exponential moving average of a list of points.
:param float smoothing_factor: the smoothing factor.
:param list points: the data points.
:return list: all ema in a list.


data = {}<EOL>if not isinstance(csv_name, (str, unicode)):<EOL><INDENT>raise exceptions.InvalidDataFormat('<STR_LIT>')<EOL><DEDENT>with open(csv_name, '<STR_LIT:r>') as csv_data:<EOL><INDENT>reader = csv.reader(csv_data, delimiter='<STR_LIT:U+002C>', quotechar='<STR_LIT:|>')<EOL>for row in reader:<EOL><INDENT>try:<EOL><INDENT>key = to_epoch(row[<NUM_LIT:0>])<EOL>value = float(row[<NUM_LIT:1>])<EOL>data[key] = value<EOL><DEDENT>except ValueError:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>return data<EOL>

Read data from a csv file into a dictionary.
:param str csv_name: path to a csv file.
:return dict: a dictionary represents the data in file.


try:<EOL><INDENT>t = float(t_str)<EOL>return t<EOL><DEDENT>except:<EOL><INDENT>for format in constants.TIMESTAMP_STR_FORMATS:<EOL><INDENT>try:<EOL><INDENT>t = datetime.datetime.strptime(t_str, format)<EOL>return float(time.mktime(t.utctimetuple()) * <NUM_LIT> + t.microsecond / <NUM_LIT>)<EOL><DEDENT>except:<EOL><INDENT>pass<EOL><DEDENT><DEDENT><DEDENT>raise exceptions.InvalidDataFormat<EOL>

Covert a timestamp string to an epoch number.
:param str t_str: a timestamp string.
:return int: epoch number of the timestamp.


def __init__(self, start_timestamp, end_timestamp, anomaly_score, exact_timestamp):

self.start_timestamp = start_timestamp<EOL>self.end_timestamp = end_timestamp<EOL>self.anomaly_score = anomaly_score<EOL>self.exact_timestamp = exact_timestamp<EOL>

Construct an anomaly object.
:param:start_timestamp: start time of the anomaly period.
:param:end_timestamp: end time of the anomaly period.
:param:anomly_score: the score of the anomaly.
:param:exact_timestamp: the timestamp within the period where the anomaly likely happened.


return self.start_timestamp, self.end_timestamp<EOL>

Get the anomaly period.
:return tuple: a tuple representation of the anomaly period.


return "<STR_LIT>".format(self.start_timestamp, self.end_timestamp, self.anomaly_score)<EOL>

return string representation of the anomaly
:return: string


return min(self.timestamps) if self.timestamps else None<EOL>

Return the earliest timestamp in the time series.


return max(self.timestamps) if self.timestamps else None<EOL>

Return the latest timestamp in the time series.


return map(lambda ts: ts * <NUM_LIT:1000>, self.timestamps)<EOL>

Return list of timestamp values in order by milliseconds since epoch.


string_rep = '<STR_LIT>'<EOL>for item in self.iteritems():<EOL><INDENT>string_rep += str(item)<EOL><DEDENT>return string_rep<EOL>

:return string: Return string representation of time series


def _generic_binary_op(self, other, op):

output = {}<EOL>if isinstance(other, TimeSeries):<EOL><INDENT>for key, value in self.items():<EOL><INDENT>if key in other:<EOL><INDENT>try:<EOL><INDENT>result = op(value, other[key])<EOL>if result is NotImplemented:<EOL><INDENT>other_type = type(other[key])<EOL>other_op = vars(other_type).get(op.__name__)<EOL>if other_op:<EOL><INDENT>output[key] = other_op(other_type(value), other[key])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>output[key] = result<EOL><DEDENT><DEDENT>except ZeroDivisionError:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for key, value in self.items():<EOL><INDENT>try:<EOL><INDENT>result = op(value, other)<EOL>if result is NotImplemented:<EOL><INDENT>other_type = type(other)<EOL>other_op = vars(other_type).get(op.__name__)<EOL>if other_op:<EOL><INDENT>output[key] = other_op(other_type(value), other)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>output[key] = result<EOL><DEDENT><DEDENT>except ZeroDivisionError:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT>if output:<EOL><INDENT>return TimeSeries(output)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>

Perform the method operation specified in the op parameter on the values
within the instance's time series values and either another time series
or a constant number value.

:param other: Time series of values or a constant number to use in calculations with instance's time series.
:param func op: The method to perform the calculation between the values.
:return: :class:`TimeSeries` object.


if self.values:<EOL><INDENT>return type(self.values[<NUM_LIT:0>])<EOL><DEDENT>elif isinstance(other, TimeSeries) and other.values:<EOL><INDENT>return type(other.values[<NUM_LIT:0>])<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>

Get the object type of the value within the values portion of the time series.

:return: `type` of object


if isinstance(other, TimeSeries):<EOL><INDENT>aligned, other_aligned = {}, {}<EOL>i, other_i = self.iteritems_silent(), other.iteritems_silent()<EOL>item, other_item = i.next(), other_i.next()<EOL>while item and other_item:<EOL><INDENT>timestamp, value = item<EOL>other_timestamp, other_value = other_item<EOL>if timestamp == other_timestamp:<EOL><INDENT>aligned[timestamp] = value<EOL>other_aligned[other_timestamp] = other_value<EOL>item = i.next()<EOL>other_item = other_i.next()<EOL><DEDENT>elif timestamp < other_timestamp:<EOL><INDENT>aligned[timestamp] = value<EOL>other_aligned[timestamp] = other_value<EOL>item = i.next()<EOL><DEDENT>else:<EOL><INDENT>aligned[other_timestamp] = value<EOL>other_aligned[other_timestamp] = other_value<EOL>other_item = other_i.next()<EOL><DEDENT><DEDENT>while item:<EOL><INDENT>timestamp, value = item<EOL>aligned[timestamp] = value<EOL>other_aligned[timestamp] = other.values[-<NUM_LIT:1>]<EOL>item = i.next()<EOL><DEDENT>while other_item:<EOL><INDENT>other_timestamp, other_value = other_item<EOL>aligned[other_timestamp] = self.values[-<NUM_LIT:1>]<EOL>other_aligned[other_timestamp] = other_value<EOL>other_item = other_i.next()<EOL><DEDENT>return TimeSeries(aligned), TimeSeries(other_aligned)<EOL><DEDENT>

Align two time series so that len(self) == len(other) and self.timstamps == other.timestamps.

:return: :tuple:(`TimeSeries` object(the aligned self), `TimeSeries` object(the aligned other))


forward_smooth = {}<EOL>backward_smooth = {}<EOL>output = {}<EOL>if self:<EOL><INDENT>pre = self.values[<NUM_LIT:0>]<EOL>next = self.values[-<NUM_LIT:1>]<EOL>for key, value in self.items():<EOL><INDENT>forward_smooth[key] = smoothing_factor * pre + (<NUM_LIT:1> - smoothing_factor) * value<EOL>pre = forward_smooth[key]<EOL><DEDENT>for key, value in reversed(self.items()):<EOL><INDENT>backward_smooth[key] = smoothing_factor * next + (<NUM_LIT:1> - smoothing_factor) * value<EOL>next = backward_smooth[key]<EOL><DEDENT>for key in forward_smooth.keys():<EOL><INDENT>output[key] = (forward_smooth[key] + backward_smooth[key]) / <NUM_LIT:2><EOL><DEDENT><DEDENT>return TimeSeries(output)<EOL>

return a new time series which is a exponential smoothed version of the original data series.
soomth forward once, backward once, and then take the average.

:param float smoothing_factor: smoothing factor
:return: :class:`TimeSeries` object.


self.timestamps = map(lambda ts: ts + offset, self.timestamps)<EOL>

Return a new time series with all timestamps incremented by some offset.

:param int offset: The number of seconds to offset the time series.
:return: `None`


maximum = self.max()<EOL>if maximum:<EOL><INDENT>self.values = map(lambda value: value / maximum, self.values)<EOL><DEDENT>

Return a new time series with all values normalized to 0 to 1.

:return: `None`


def crop(self, start_timestamp, end_timestamp):

output = {}<EOL>for key, value in self.items():<EOL><INDENT>if key >= start_timestamp and key <= end_timestamp:<EOL><INDENT>output[key] = value<EOL><DEDENT><DEDENT>if output:<EOL><INDENT>return TimeSeries(output)<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>')<EOL><DEDENT>

Return a new TimeSeries object contains all the timstamps and values within
the specified range.

:param int start_timestamp: the start timestamp value
:param int end_timestamp: the end timestamp value
:return: :class:`TimeSeries` object.


return numpy.asscalar(numpy.average(self.values)) if self.values else default<EOL>

Calculate the average value over the time series.

:param default: Value to return as a default should the calculation not be possible.
:return: Float representing the average value or `None`.


return numpy.asscalar(numpy.median(self.values)) if self.values else default<EOL>

Calculate the median value over the time series.

:param default: Value to return as a default should the calculation not be possible.
:return: Float representing the median value or `None`.


return numpy.asscalar(numpy.max(self.values)) if self.values else default<EOL>

Calculate the maximum value over the time series.

:param default: Value to return as a default should the calculation not be possible.
:return: Float representing the maximum value or `None`.


return numpy.asscalar(numpy.min(self.values)) if self.values else default<EOL>

Calculate the minimum value over the time series.

:param default: Value to return as a default should the calculation not be possible.
:return: Float representing the maximum value or `None`.


return numpy.asscalar(numpy.percentile(self.values, n)) if self.values else default<EOL>

Calculate the Nth Percentile value over the time series.

:param int n: Integer value of the percentile to calculate.
:param default: Value to return as a default should the calculation not be possible.
:return: Float representing the Nth percentile value or `None`.


return numpy.asscalar(numpy.std(self.values)) if self.values else default<EOL>

Calculate the standard deviation of the time series.

:param default: Value to return as a default should the calculation not be possible.
:return: Float representing the standard deviation value or `None`.


return numpy.asscalar(numpy.sum(self.values)) if self.values else default<EOL>

Calculate the sum of all the values in the times series.

:param default: Value to return as a default should the calculation not be possible.
:return: Float representing the sum or `None`.


def __init__(self, shift, coefficient, shifted_coefficient):

self.shift = shift<EOL>self.coefficient = coefficient<EOL>self.shifted_coefficient = shifted_coefficient<EOL>

Construct a CorrelationResult object.
:param int shift: the amount of shift where the coefficient is obtained.
:param float coefficient: the correlation coefficient.
:param float shifted_coefficient: the correlation coefficient with shift taken into account.


def graph_csv(output_directory, resource_path, csv_file, plot_title, output_filename, y_label=None, precision=None, graph_height="<STR_LIT>", graph_width="<STR_LIT>"):

if not os.path.getsize(csv_file):<EOL><INDENT>return False, "<STR_LIT>"<EOL><DEDENT>y_label = y_label or plot_title<EOL>div_id = str(random.random())<EOL>div_string = "<STR_LIT>" % (div_id, graph_width, graph_height)<EOL>script_string = """<STR_LIT>""" + div_id + """<STR_LIT>""" + resource_path + '<STR_LIT:/>' + os.path.basename(csv_file) + """<STR_LIT>""" + y_label + """<STR_LIT>""" + plot_title + """<STR_LIT>""" + y_label + """<STR_LIT>"""<EOL>with open(os.path.join(output_directory, output_filename + '<STR_LIT>'), '<STR_LIT:w>') as div_file:<EOL><INDENT>div_file.write(div_string + script_string)<EOL><DEDENT>return True, os.path.join(output_directory, output_filename + '<STR_LIT>')<EOL>


def graph_data_on_the_same_graph(list_of_plots, output_directory, resource_path, output_filename):

logger.warning('<STR_LIT>')<EOL>return False, None<EOL>

graph_data_on_the_same_graph: put a list of plots on the same graph: currently it supports CDF


def highlight_region(plt, start_x, end_x):

start_x = convert_to_mdate(start_x)<EOL>end_x = convert_to_mdate(end_x)<EOL>plt.axvspan(start_x, end_x, color=CONSTANTS.HIGHLIGHT_COLOR, alpha=CONSTANTS.HIGHLIGHT_ALPHA)<EOL>

Highlight a region on the chart between the specified start and end x-co-ordinates.
param pyplot plt: matplotlibk pyplot which contains the charts to be highlighted
param string start_x : epoch time millis
param string end_x : epoch time millis


maximum_yvalue = -float('<STR_LIT>')<EOL>minimum_yvalue = float('<STR_LIT>')<EOL>plots = curate_plot_list(list_of_plots)<EOL>plot_count = len(plots)<EOL>if plot_count == <NUM_LIT:0>:<EOL><INDENT>return False, None<EOL><DEDENT>graph_height, graph_width, graph_title = get_graph_metadata(plots)<EOL>current_plot_count = <NUM_LIT:0><EOL>fig, axis = plt.subplots()<EOL>fig.set_size_inches(graph_width, graph_height)<EOL>if plot_count < <NUM_LIT:2>:<EOL><INDENT>fig.subplots_adjust(left=CONSTANTS.SUBPLOT_LEFT_OFFSET, bottom=CONSTANTS.SUBPLOT_BOTTOM_OFFSET, right=CONSTANTS.SUBPLOT_RIGHT_OFFSET)<EOL><DEDENT>else:<EOL><INDENT>fig.subplots_adjust(left=CONSTANTS.SUBPLOT_LEFT_OFFSET, bottom=CONSTANTS.SUBPLOT_BOTTOM_OFFSET,<EOL>right=CONSTANTS.SUBPLOT_RIGHT_OFFSET - CONSTANTS.Y_AXIS_OFFSET * (plot_count - <NUM_LIT:2>))<EOL><DEDENT>for plot in plots:<EOL><INDENT>current_plot_count += <NUM_LIT:1><EOL>logger.info('<STR_LIT>' + plot.input_csv + '<STR_LIT>' + output_filename + '<STR_LIT>')<EOL>xval, yval = numpy.loadtxt(plot.input_csv, unpack=True, delimiter='<STR_LIT:U+002C>')<EOL>axis.plot(xval, yval, linestyle='<STR_LIT:->', marker=None, color=get_current_color(current_plot_count), label=plot.plot_label)<EOL>axis.legend()<EOL>maximum_yvalue = max(maximum_yvalue, numpy.amax(yval) * (<NUM_LIT:1.0> + CONSTANTS.ZOOM_FACTOR * current_plot_count))<EOL>minimum_yvalue = min(minimum_yvalue, numpy.amin(yval) * (<NUM_LIT:1.0> - CONSTANTS.ZOOM_FACTOR * current_plot_count))<EOL><DEDENT>axis.yaxis.set_ticks_position('<STR_LIT:left>')<EOL>axis.set_xlabel(plots[<NUM_LIT:0>].x_label)<EOL>axis.set_ylabel(plots[<NUM_LIT:0>].y_label, fontsize=CONSTANTS.Y_LABEL_FONTSIZE)<EOL>axis.set_ylim([minimum_yvalue, maximum_yvalue])<EOL>axis.yaxis.grid(True)<EOL>axis.xaxis.grid(True)<EOL>axis.set_title(graph_title)<EOL>plot_file_name = os.path.join(output_directory, output_filename + "<STR_LIT>")<EOL>fig.savefig(plot_file_name)<EOL>plt.close()<EOL>with open(os.path.join(output_directory, output_filename + '<STR_LIT>'), '<STR_LIT:w>') as div_file:<EOL><INDENT>div_file.write('<STR_LIT>' + os.path.basename(plot_file_name).replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>") + '<STR_LIT>' +<EOL>resource_path + '<STR_LIT:/>' + os.path.basename(plot_file_name) + '<STR_LIT>' + os.path.basename(plot_file_name) +<EOL>'<STR_LIT>' + os.path.basename(plot_file_name) + '<STR_LIT>')<EOL><DEDENT>return True, os.path.join(output_directory, output_filename + '<STR_LIT>')<EOL>


return naarad.resources.get_dir()<EOL>

Return the location of the report templates and local js/css includes.
:return: filesystem location for naarad/src/resources directory.


resource_folder = self.get_resources_location()<EOL>for stylesheet in self.stylesheet_includes:<EOL><INDENT>if ('<STR_LIT:http>' not in stylesheet) and naarad.utils.is_valid_file(os.path.join(resource_folder, stylesheet)):<EOL><INDENT>shutil.copy(os.path.join(resource_folder, stylesheet), self.resource_directory)<EOL><DEDENT><DEDENT>for javascript in self.javascript_includes:<EOL><INDENT>if ('<STR_LIT:http>' not in javascript) and naarad.utils.is_valid_file(os.path.join(resource_folder, javascript)):<EOL><INDENT>shutil.copy(os.path.join(resource_folder, javascript), self.resource_directory)<EOL><DEDENT><DEDENT>return None<EOL>

Copy local js/css includes from naarad resources to the report/resources directory
:return: None


if not os.path.exists(self.resource_directory):<EOL><INDENT>os.makedirs(self.resource_directory)<EOL><DEDENT>self.copy_local_includes()<EOL>template_loader = FileSystemLoader(self.get_resources_location())<EOL>template_environment = Environment(loader=template_loader)<EOL>client_html = template_environment.get_template(CONSTANTS.TEMPLATE_HEADER).render(custom_stylesheet_includes=CONSTANTS.STYLESHEET_INCLUDES,<EOL>custom_javascript_includes=CONSTANTS.JAVASCRIPT_INCLUDES,<EOL>resource_path=self.resource_path,<EOL>report_title='<STR_LIT>') + '<STR_LIT:\n>'<EOL>client_html += template_environment.get_template(CONSTANTS.TEMPLATE_DIFF_CLIENT_CHARTING).render(data_series=data_sources,<EOL>resource_path=self.resource_path) + '<STR_LIT:\n>'<EOL>client_html += template_environment.get_template(CONSTANTS.TEMPLATE_FOOTER).render()<EOL>return client_html<EOL>

Create the client charting page for the diff report, with time series data from the two diffed reports.
:return: generated html to be written to disk


if not os.path.exists(self.resource_directory):<EOL><INDENT>os.makedirs(self.resource_directory)<EOL><DEDENT>self.copy_local_includes()<EOL>div_html = '<STR_LIT>'<EOL>for plot_div in sorted(self.plot_files):<EOL><INDENT>with open(plot_div, '<STR_LIT:r>') as div_file:<EOL><INDENT>div_html += '<STR_LIT:\n>' + div_file.read()<EOL><DEDENT><DEDENT>template_loader = FileSystemLoader(self.get_resources_location())<EOL>template_environment = Environment(loader=template_loader)<EOL>template_environment.filters['<STR_LIT>'] = naarad.utils.sanitize_string<EOL>diff_html = template_environment.get_template(CONSTANTS.TEMPLATE_HEADER).render(custom_stylesheet_includes=CONSTANTS.STYLESHEET_INCLUDES,<EOL>custom_javascript_includes=CONSTANTS.JAVASCRIPT_INCLUDES,<EOL>resource_path=self.resource_path,<EOL>report_title='<STR_LIT>') + '<STR_LIT:\n>'<EOL>diff_html += template_environment.get_template(CONSTANTS.TEMPLATE_DIFF_PAGE).render(diff_data=self.diff_data, plot_div_content=div_html,<EOL>reports=self.reports, sla_failure_list=self.sla_failure_list,<EOL>sla_map=self.sla_map) + '<STR_LIT:\n>'<EOL>diff_html += template_environment.get_template(CONSTANTS.TEMPLATE_FOOTER).render()<EOL>return diff_html<EOL>

Generate the summary diff report html from template
:return: generated html to be written to disk


for report in self.reports:<EOL><INDENT>if report.remote_location == '<STR_LIT>':<EOL><INDENT>if naarad.utils.is_valid_file(os.path.join(os.path.join(report.location, self.resource_path), metafile)):<EOL><INDENT>with open(os.path.join(os.path.join(report.location, self.resource_path), metafile), '<STR_LIT:r>') as meta_file:<EOL><INDENT>if metafile == CONSTANTS.STATS_CSV_LIST_FILE:<EOL><INDENT>report.stats = meta_file.readlines()[<NUM_LIT:0>].split('<STR_LIT:U+002C>')<EOL><DEDENT>elif metafile == CONSTANTS.PLOTS_CSV_LIST_FILE:<EOL><INDENT>report.datasource = meta_file.readlines()[<NUM_LIT:0>].split('<STR_LIT:U+002C>')<EOL><DEDENT>elif metafile == CONSTANTS.CDF_PLOTS_CSV_LIST_FILE:<EOL><INDENT>report.cdf_datasource = meta_file.readlines()[<NUM_LIT:0>].split('<STR_LIT:U+002C>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>report.status = '<STR_LIT>'<EOL>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>', report.label)<EOL>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>stats_url = report.remote_location + '<STR_LIT:/>' + self.resource_path + '<STR_LIT:/>' + metafile<EOL>meta_file_data = naarad.httpdownload.stream_url(stats_url)<EOL>if meta_file_data:<EOL><INDENT>if metafile == CONSTANTS.STATS_CSV_LIST_FILE:<EOL><INDENT>report.stats = meta_file_data.split('<STR_LIT:U+002C>')<EOL><DEDENT>elif metafile == CONSTANTS.PLOTS_CSV_LIST_FILE:<EOL><INDENT>report.datasource = meta_file_data.split('<STR_LIT:U+002C>')<EOL><DEDENT>elif metafile == CONSTANTS.CDF_PLOTS_CSV_LIST_FILE:<EOL><INDENT>report.cdf_datasource = meta_file_data.split('<STR_LIT:U+002C>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>report.status = '<STR_LIT>'<EOL>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>', report.label)<EOL>return False<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Determine what summary stats, time series, and CDF csv exist for the reports that need to be diffed.
:return: boolean: return whether the summary stats / time series / CDF csv summary was successfully located


report_count = <NUM_LIT:0><EOL>if self.status != '<STR_LIT:OK>':<EOL><INDENT>return False<EOL><DEDENT>diff_datasource = sorted(set(self.reports[<NUM_LIT:0>].datasource) & set(self.reports[<NUM_LIT:1>].datasource))<EOL>if diff_datasource:<EOL><INDENT>self.reports[<NUM_LIT:0>].datasource = diff_datasource<EOL>self.reports[<NUM_LIT:1>].datasource = diff_datasource<EOL><DEDENT>else:<EOL><INDENT>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>')<EOL>return False<EOL><DEDENT>for report in self.reports:<EOL><INDENT>report.label = report_count<EOL>report_count += <NUM_LIT:1><EOL>report.local_location = os.path.join(self.resource_directory, str(report.label))<EOL>try:<EOL><INDENT>os.makedirs(report.local_location)<EOL><DEDENT>except OSError as exeption:<EOL><INDENT>if exeption.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>if report.remote_location != '<STR_LIT>':<EOL><INDENT>naarad.httpdownload.download_url_list(map(lambda x: report.remote_location + '<STR_LIT:/>' + self.resource_path + '<STR_LIT:/>' + x + '<STR_LIT>', report.datasource),<EOL>report.local_location)<EOL><DEDENT>else:<EOL><INDENT>for filename in report.datasource:<EOL><INDENT>try:<EOL><INDENT>shutil.copy(os.path.join(os.path.join(report.location, self.resource_path), filename + '<STR_LIT>'), report.local_location)<EOL><DEDENT>except IOError as exeption:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return True<EOL>

Identify what time series exist in both the diffed reports and download them to the diff report resources directory
:return: True/False : return status of whether the download of time series resources succeeded.


report_count = <NUM_LIT:0><EOL>if self.status != '<STR_LIT:OK>':<EOL><INDENT>return False<EOL><DEDENT>diff_cdf_datasource = sorted(set(self.reports[<NUM_LIT:0>].cdf_datasource) & set(self.reports[<NUM_LIT:1>].cdf_datasource))<EOL>if diff_cdf_datasource:<EOL><INDENT>self.reports[<NUM_LIT:0>].cdf_datasource = diff_cdf_datasource<EOL>self.reports[<NUM_LIT:1>].cdf_datasource = diff_cdf_datasource<EOL><DEDENT>else:<EOL><INDENT>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>')<EOL>return False<EOL><DEDENT>for report in self.reports:<EOL><INDENT>report.label = report_count<EOL>report_count += <NUM_LIT:1><EOL>report.local_location = os.path.join(self.resource_directory, str(report.label))<EOL>try:<EOL><INDENT>os.makedirs(report.local_location)<EOL><DEDENT>except OSError as exeption:<EOL><INDENT>if exeption.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>if report.remote_location != '<STR_LIT>':<EOL><INDENT>naarad.httpdownload.download_url_list(map(lambda x: report.remote_location + '<STR_LIT:/>' + self.resource_path + '<STR_LIT:/>' + x + '<STR_LIT>', report.cdf_datasource),<EOL>report.local_location)<EOL><DEDENT>else:<EOL><INDENT>for filename in report.cdf_datasource:<EOL><INDENT>try:<EOL><INDENT>shutil.copy(os.path.join(os.path.join(report.location, self.resource_path), filename + '<STR_LIT>'), report.local_location)<EOL><DEDENT>except IOError as exeption:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return True<EOL>

Identify what cdf series exist in both the diffed reports and download them to the diff report resources directory
:return: True/False : return status of whether the download of time series resources succeeded.


report_count = <NUM_LIT:0><EOL>if self.status != '<STR_LIT:OK>':<EOL><INDENT>return False<EOL><DEDENT>diff_stats = set(self.reports[<NUM_LIT:0>].stats) & set(self.reports[<NUM_LIT:1>].stats)<EOL>if diff_stats:<EOL><INDENT>self.reports[<NUM_LIT:0>].stats = diff_stats<EOL>self.reports[<NUM_LIT:1>].stats = diff_stats<EOL><DEDENT>else:<EOL><INDENT>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>')<EOL>return False<EOL><DEDENT>for report in self.reports:<EOL><INDENT>report.label = report_count<EOL>report_count += <NUM_LIT:1><EOL>report.local_location = os.path.join(self.resource_directory, str(report.label))<EOL>try:<EOL><INDENT>os.makedirs(report.local_location)<EOL><DEDENT>except OSError as exeption:<EOL><INDENT>if exeption.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>if report.remote_location != '<STR_LIT>':<EOL><INDENT>naarad.httpdownload.download_url_list(map(lambda x: report.remote_location + '<STR_LIT:/>' + self.resource_path + '<STR_LIT:/>' + x, report.stats), report.local_location)<EOL><DEDENT>else:<EOL><INDENT>for filename in report.stats:<EOL><INDENT>shutil.copy(os.path.join(os.path.join(report.location, self.resource_path), filename), report.local_location)<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Identify what summary stats exist in both the diffed reports and download them to the diff report resources directory
:return: True/False : return status of whether the download of summary stats succeeded.


diff_datasource = sorted(set(self.reports[<NUM_LIT:0>].datasource) & set(self.reports[<NUM_LIT:1>].datasource))<EOL>graphed = False<EOL>for submetric in diff_datasource:<EOL><INDENT>baseline_csv = naarad.utils.get_default_csv(self.reports[<NUM_LIT:0>].local_location, (submetric + '<STR_LIT>'))<EOL>current_csv = naarad.utils.get_default_csv(self.reports[<NUM_LIT:1>].local_location, (submetric + '<STR_LIT>'))<EOL>if (not (naarad.utils.is_valid_file(baseline_csv) & naarad.utils.is_valid_file(current_csv))):<EOL><INDENT>continue<EOL><DEDENT>baseline_plot = PD(input_csv=baseline_csv, csv_column=<NUM_LIT:1>, series_name=submetric, y_label=submetric, precision=None, graph_height=<NUM_LIT>, graph_width=<NUM_LIT>,<EOL>graph_type='<STR_LIT>', plot_label='<STR_LIT>', x_label='<STR_LIT>')<EOL>current_plot = PD(input_csv=current_csv, csv_column=<NUM_LIT:1>, series_name=submetric, y_label=submetric, precision=None, graph_height=<NUM_LIT>, graph_width=<NUM_LIT>,<EOL>graph_type='<STR_LIT>', plot_label='<STR_LIT>', x_label='<STR_LIT>')<EOL>graphed, div_file = Diff.graphing_modules[graphing_library].graph_data_on_the_same_graph([baseline_plot, current_plot],<EOL>os.path.join(self.output_directory, self.resource_path),<EOL>self.resource_path, (submetric + '<STR_LIT>'))<EOL>if graphed:<EOL><INDENT>self.plot_files.append(div_file)<EOL><DEDENT><DEDENT>return True<EOL>

Generate CDF diff plots of the submetrics


try:<EOL><INDENT>if sla.display is '<STR_LIT:%>':<EOL><INDENT>diff_val = float(diff_metric['<STR_LIT>'])<EOL><DEDENT>else:<EOL><INDENT>diff_val = float(diff_metric['<STR_LIT>'])<EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>return False<EOL><DEDENT>if not (sla.check_sla_passed(diff_val)):<EOL><INDENT>self.sla_failures += <NUM_LIT:1><EOL>self.sla_failure_list.append(DiffSLAFailure(sla, diff_metric))<EOL><DEDENT>return True<EOL>

Check whether the SLA has passed or failed


if (self.discover(CONSTANTS.STATS_CSV_LIST_FILE) and self.discover(CONSTANTS.PLOTS_CSV_LIST_FILE) and self.discover(CONSTANTS.CDF_PLOTS_CSV_LIST_FILE) and<EOL>self.collect() and self.collect_datasources() and self.collect_cdf_datasources()):<EOL><INDENT>for stats in self.reports[<NUM_LIT:0>].stats:<EOL><INDENT>metric_label = stats.replace('<STR_LIT>', '<STR_LIT>')<EOL>stats_0 = os.path.join(self.reports[<NUM_LIT:0>].local_location, stats)<EOL>stats_1 = os.path.join(self.reports[<NUM_LIT:1>].local_location, stats)<EOL>report0_stats = {}<EOL>report1_stats = {}<EOL>if naarad.utils.is_valid_file(stats_0) and naarad.utils.is_valid_file(stats_1):<EOL><INDENT>report0 = csv.DictReader(open(stats_0))<EOL>for row in report0:<EOL><INDENT>report0_stats[row[CONSTANTS.SUBMETRIC_HEADER]] = row<EOL><DEDENT>report0_stats['<STR_LIT>'] = report0._fieldnames<EOL>report1 = csv.DictReader(open(stats_1))<EOL>for row in report1:<EOL><INDENT>report1_stats[row[CONSTANTS.SUBMETRIC_HEADER]] = row<EOL><DEDENT>report1_stats['<STR_LIT>'] = report1._fieldnames<EOL>common_stats = sorted(set(report0_stats['<STR_LIT>']) & set(report1_stats['<STR_LIT>']))<EOL>common_submetrics = sorted(set(report0_stats.keys()) & set(report1_stats.keys()))<EOL>for submetric in common_submetrics:<EOL><INDENT>if submetric != '<STR_LIT>':<EOL><INDENT>for stat in common_stats:<EOL><INDENT>if stat != CONSTANTS.SUBMETRIC_HEADER:<EOL><INDENT>diff_metric = reduce(defaultdict.__getitem__, [stats.split('<STR_LIT:.>')[<NUM_LIT:0>], submetric, stat], self.diff_data)<EOL>diff_metric[<NUM_LIT:0>] = float(report0_stats[submetric][stat])<EOL>diff_metric[<NUM_LIT:1>] = float(report1_stats[submetric][stat])<EOL>diff_metric['<STR_LIT>'] = naarad.utils.normalize_float_for_display(diff_metric[<NUM_LIT:1>] - diff_metric[<NUM_LIT:0>])<EOL>if diff_metric[<NUM_LIT:0>] == <NUM_LIT:0>:<EOL><INDENT>if diff_metric['<STR_LIT>'] == '<STR_LIT>':<EOL><INDENT>diff_metric['<STR_LIT>'] = <NUM_LIT:0.0><EOL><DEDENT>else:<EOL><INDENT>diff_metric['<STR_LIT>'] = '<STR_LIT>'<EOL><DEDENT><DEDENT>else:<EOL><INDENT>diff_metric['<STR_LIT>'] = naarad.utils.normalize_float_for_display((diff_metric[<NUM_LIT:1>] - diff_metric[<NUM_LIT:0>]) * <NUM_LIT:100> / diff_metric[<NUM_LIT:0>])<EOL><DEDENT>if ((metric_label in self.sla_map.keys()) and (submetric in self.sla_map[metric_label].keys()) and<EOL>(stat in self.sla_map[metric_label][submetric].keys())):<EOL><INDENT>self.check_sla(self.sla_map[metric_label][submetric][stat], diff_metric)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>self.plot_diff()<EOL>diff_html = '<STR_LIT>'<EOL>if self.diff_data:<EOL><INDENT>diff_html = self.generate_diff_html()<EOL>client_html = self.generate_client_charting_page(self.reports[<NUM_LIT:0>].datasource)<EOL><DEDENT>if diff_html != '<STR_LIT>':<EOL><INDENT>with open(os.path.join(self.output_directory, CONSTANTS.DIFF_REPORT_FILE), '<STR_LIT:w>') as diff_file:<EOL><INDENT>diff_file.write(diff_html)<EOL><DEDENT>with open(os.path.join(self.output_directory, CONSTANTS.CLIENT_CHARTING_FILE), '<STR_LIT:w>') as client_file:<EOL><INDENT>client_file.write(client_html)<EOL><DEDENT><DEDENT>return True<EOL>

Generate a diff report from the reports specified.
:return: True/False : return status of whether the diff report generation succeeded.


for col, value in values.items():<EOL><INDENT>if col in self.column_csv_map:<EOL><INDENT>out_csv = self.column_csv_map[col]<EOL><DEDENT>else:<EOL><INDENT>out_csv = self.get_csv(col)   <EOL>self.data[out_csv] = []<EOL><DEDENT>self.data[out_csv].append(self.ts + "<STR_LIT:U+002C>" + value)<EOL><DEDENT>

Take the (col, value) in 'values', append value into 'col' in self.data[]


self.ts_time = words[<NUM_LIT:2>]<EOL>self.ts = self.ts_date + '<STR_LIT:U+0020>' + self.ts_time<EOL>self.ts = ts = naarad.utils.get_standardized_timestamp(self.ts, None)<EOL>if self.ts_out_of_range(self.ts):<EOL><INDENT>self.ts_valid_lines = False<EOL><DEDENT>else:<EOL><INDENT>self.ts_valid_lines = True<EOL><DEDENT>up_days = int(words[<NUM_LIT:4>])<EOL>up_hour_minute = words[<NUM_LIT:6>].split('<STR_LIT::>')  <EOL>up_minutes = int(up_hour_minute[<NUM_LIT:0>]) * <NUM_LIT> + int(up_hour_minute[<NUM_LIT:1>].split('<STR_LIT:U+002C>')[<NUM_LIT:0>])<EOL>uptime_minute = up_days * <NUM_LIT> * <NUM_LIT> + up_minutes  <EOL>values = {}<EOL>values['<STR_LIT>'] = str(uptime_minute)<EOL>values['<STR_LIT>'] = words[<NUM_LIT:7>]<EOL>values['<STR_LIT>'] = words[<NUM_LIT:11>][:-<NUM_LIT:1>]<EOL>values['<STR_LIT>'] = words[<NUM_LIT:12>][:-<NUM_LIT:1>]<EOL>values['<STR_LIT>'] = words[<NUM_LIT>]<EOL>self.put_values_into_data(values)<EOL>

Process the line starting with "top"
Example log:   top - 00:00:02 up 32 days,  7:08, 19 users,  load average: 0.00, 0.00, 0.00


words = words[<NUM_LIT:1>:]<EOL>length = len(words) / <NUM_LIT:2>  <EOL>values = {}<EOL>for offset in range(length):<EOL><INDENT>k = words[<NUM_LIT:2> * offset + <NUM_LIT:1>].strip('<STR_LIT:U+002C>')<EOL>v = words[<NUM_LIT:2> * offset]<EOL>values['<STR_LIT>' + k] = v<EOL><DEDENT>self.put_values_into_data(values)<EOL>

Process the line starting with "Tasks:"
Example log:   Tasks: 446 total,   1 running, 442 sleeping,   2 stopped,   1 zombie


values = {}<EOL>for word in words[<NUM_LIT:1>:]:<EOL><INDENT>val, key = word.split('<STR_LIT:%>')<EOL>values['<STR_LIT>' + key.strip('<STR_LIT:U+002C>')] = val<EOL><DEDENT>self.put_values_into_data(values)<EOL>

Process the line starting with "Cpu(s):"
Example log: Cpu(s):  1.3%us,  0.5%sy,  0.0%ni, 98.2%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st


value = <NUM_LIT:0.0><EOL>if word[-<NUM_LIT:1>] == '<STR_LIT>' or word[-<NUM_LIT:1>] == '<STR_LIT:g>':<EOL><INDENT>value = float(word[:-<NUM_LIT:1>])<EOL><DEDENT>elif word[-<NUM_LIT:1>] == '<STR_LIT:M>' or word[-<NUM_LIT:1>] == '<STR_LIT:m>':<EOL><INDENT>value = float(word[:-<NUM_LIT:1>]) / <NUM_LIT><EOL><DEDENT>elif word[-<NUM_LIT:1>] == '<STR_LIT>' or word[-<NUM_LIT:1>] == '<STR_LIT:k>':<EOL><INDENT>value = float(word[:-<NUM_LIT:1>]) / <NUM_LIT> / <NUM_LIT><EOL><DEDENT>else:  <EOL><INDENT>value = float(word) / <NUM_LIT> / <NUM_LIT> / <NUM_LIT><EOL><DEDENT>return str(value)<EOL>

Given a size such as '2333M', return the converted value in G


words = words[<NUM_LIT:1>:]<EOL>length = len(words) / <NUM_LIT:2>  <EOL>values = {}<EOL>for offset in range(length):<EOL><INDENT>k = words[<NUM_LIT:2> * offset + <NUM_LIT:1>].strip('<STR_LIT:U+002C>')<EOL>v = self.convert_to_G(words[<NUM_LIT:2> * offset])<EOL>values['<STR_LIT>' + k] = v<EOL><DEDENT>self.put_values_into_data(values)<EOL>

Process the line starting with "Mem:"
Example log: Mem:    62.841G total,   16.038G used,   46.803G free,  650.312M buffers
For each value, needs to convert to 'G' (needs to handle cases of K, M)


Process the line starting with "Swap:"
Example log: Swap:   63.998G total,    0.000k used,   63.998G free,   11.324G cached
For each value, needs to convert to 'G' (needs to handle cases of K, M)


pid_index = self.process_headers.index('<STR_LIT>')<EOL>proces_index = self.process_headers.index('<STR_LIT>')<EOL>pid = words[pid_index]<EOL>process = words[proces_index]<EOL>if pid in self.PID or process in self.COMMAND:<EOL><INDENT>process_name = process.split('<STR_LIT:/>')[<NUM_LIT:0>]<EOL>values = {}<EOL>for word_col in self.process_headers:<EOL><INDENT>word_index = self.process_headers.index(word_col)<EOL>if word_col in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:  <EOL><INDENT>values[process_name + '<STR_LIT:_>' + pid + '<STR_LIT:_>' + word_col] = self.convert_to_G(words[word_index])<EOL><DEDENT>elif word_col in ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']:  <EOL><INDENT>values[process_name + '<STR_LIT:_>' + pid + '<STR_LIT:_>' + word_col.strip('<STR_LIT:%>')] = words[word_index]<EOL><DEDENT>uptime_index = self.process_headers.index('<STR_LIT>')<EOL>uptime = words[uptime_index].split('<STR_LIT::>')<EOL>uptime_sec = float(uptime[<NUM_LIT:0>]) * <NUM_LIT> + float(uptime[<NUM_LIT:1>])<EOL>values[process_name + '<STR_LIT:_>' + pid + '<STR_LIT:_>' + '<STR_LIT>'] = str(uptime_sec)<EOL><DEDENT>self.put_values_into_data(values)<EOL><DEDENT>

process the individual lines like this:
#PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
29303 root      20   0 35300 2580 1664 R  3.9  0.0   0:00.02 top
11 root      RT   0     0    0    0 S  1.9  0.0   0:18.87 migration/2
3702 root      20   0 34884 4192 1692 S  1.9  0.0  31:40.47 cf-serverd
It does not record all processes due to memory concern; rather only records interested processes (based on user input of PID and COMMAND)


for infile in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', infile)<EOL>status = True<EOL>file_status = naarad.utils.is_valid_file(infile)<EOL>if not file_status:<EOL><INDENT>return False<EOL><DEDENT>with open(infile) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>words = line.split()<EOL>if not words:<EOL><INDENT>continue<EOL><DEDENT>if re.match('<STR_LIT>', line):<EOL><INDENT>self.ts_date = words[<NUM_LIT:0>]<EOL>continue<EOL><DEDENT>prefix_word = words[<NUM_LIT:0>].strip()<EOL>if prefix_word == '<STR_LIT>':<EOL><INDENT>self.process_top_line(words)<EOL>self.saw_pid = False  <EOL><DEDENT>elif self.ts_valid_lines:<EOL><INDENT>if prefix_word == '<STR_LIT>':<EOL><INDENT>self.process_tasks_line(words)<EOL><DEDENT>elif prefix_word == '<STR_LIT>':<EOL><INDENT>self.process_cpu_line(words)<EOL><DEDENT>elif prefix_word == '<STR_LIT>':<EOL><INDENT>self.process_mem_line(words)<EOL><DEDENT>elif prefix_word == '<STR_LIT>':<EOL><INDENT>self.process_swap_line(words)<EOL><DEDENT>elif prefix_word == '<STR_LIT>':<EOL><INDENT>self.saw_pid = True  <EOL>self.process_headers = words<EOL><DEDENT>else:  <EOL><INDENT>if self.saw_pid and len(words) >= len(self.process_headers):  <EOL><INDENT>self.process_individual_command(words)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>for out_csv in self.data.keys():    <EOL><INDENT>self.csv_files.append(out_csv)<EOL>with open(out_csv, '<STR_LIT:w>') as fh:<EOL><INDENT>fh.write('<STR_LIT:\n>'.join(self.data[out_csv]))<EOL><DEDENT><DEDENT>gc.collect()<EOL>return status<EOL>

Parse the top output file
Return status of the metric parse

The raw log file is like the following:
2014-06-23
top - 00:00:02 up 18 days,  7:08, 19 users,  load average: 0.05, 0.03, 0.00
Tasks: 447 total,   1 running, 443 sleeping,   2 stopped,   1 zombie
Cpu(s):  1.6%us,  0.5%sy,  0.0%ni, 97.9%id,  0.0%wa,  0.0%hi,  0.0%si,  0.0%st
Mem:    62.841G total,   15.167G used,   47.675G free,  643.434M buffers
Swap:   63.998G total,    0.000k used,   63.998G free,   11.324G cached

PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
1730 root      20   0 4457m  10m 3328 S  1.9  0.0  80:13.45 lwregd
The log lines can be generated by echo $t >> $RESULT/top.out &; top -b -n $COUNT -d $INTERVAL | grep -A 40 '^top' >> $RESULT/top.out &


for aggr_metric in self.aggr_metrics:   <EOL><INDENT>functions_aggr = []<EOL>fields = aggr_metric.split("<STR_LIT::>")<EOL>cur_metric_type = fields[<NUM_LIT:0>].split("<STR_LIT:.>")[<NUM_LIT:0>]  <EOL>if len(fields) > <NUM_LIT:1>:  <EOL><INDENT>func_user = '<STR_LIT>'.join(fields[<NUM_LIT:1>].split())<EOL>functions_aggr.extend(func_user.split("<STR_LIT:U+002C>"))<EOL><DEDENT>else:  <EOL><INDENT>return True<EOL><DEDENT>cur_column = '<STR_LIT:.>'.join(fields[<NUM_LIT:0>].split('<STR_LIT:.>')[<NUM_LIT:1>:])    <EOL>aggr_data = {}<EOL>aggr_data['<STR_LIT>'] = []   <EOL>aggr_data['<STR_LIT>'] = defaultdict(float)   <EOL>aggr_data['<STR_LIT:count>'] = defaultdict(int)  <EOL>for metric in self.metrics:   <EOL><INDENT>if metric.hostname in self.aggr_hosts andcur_column in metric.csv_column_map.values():<EOL><INDENT>file_csv = metric.get_csv(cur_column)<EOL>timestamp_format = None<EOL>with open(file_csv) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>aggr_data['<STR_LIT>'].append(line.rstrip())<EOL>words = line.split("<STR_LIT:U+002C>")<EOL>ts = words[<NUM_LIT:0>].split('<STR_LIT:.>')[<NUM_LIT:0>]   <EOL>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(ts)<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(ts, timestamp_format)<EOL>aggr_data['<STR_LIT>'][ts] += float(words[<NUM_LIT:1>])<EOL>aggr_data['<STR_LIT:count>'][ts] += <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in functions_aggr:<EOL><INDENT>out_csv = self.get_csv(cur_column, '<STR_LIT>')<EOL>self.csv_files.append(out_csv)<EOL>with open(out_csv, '<STR_LIT:w>') as fh:<EOL><INDENT>fh.write("<STR_LIT:\n>".join(sorted(aggr_data['<STR_LIT>'])))<EOL><DEDENT><DEDENT>if '<STR_LIT>' in functions_aggr:<EOL><INDENT>out_csv = self.get_csv(cur_column, '<STR_LIT>')<EOL>self.csv_files.append(out_csv)<EOL>with open(out_csv, '<STR_LIT:w>') as fh:<EOL><INDENT>for (k, v) in sorted(aggr_data['<STR_LIT>'].items()):<EOL><INDENT>fh.write(k + "<STR_LIT:U+002C>" + str(v) + '<STR_LIT:\n>')<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT>' in functions_aggr:<EOL><INDENT>out_csv = self.get_csv(cur_column, '<STR_LIT>')<EOL>self.csv_files.append(out_csv)<EOL>with open(out_csv, '<STR_LIT:w>') as fh:<EOL><INDENT>for (k, v) in sorted(aggr_data['<STR_LIT>'].items()):<EOL><INDENT>fh.write(k + "<STR_LIT:U+002C>" + str(v / aggr_data['<STR_LIT:count>'][k]) + '<STR_LIT:\n>')<EOL><DEDENT><DEDENT><DEDENT>if '<STR_LIT:count>' in functions_aggr:<EOL><INDENT>out_csv = self.get_csv(cur_column, '<STR_LIT:count>')<EOL>self.csv_files.append(out_csv)<EOL>with open(out_csv, '<STR_LIT:w>') as fh:<EOL><INDENT>for (k, v) in sorted(aggr_data['<STR_LIT:count>'].items()):<EOL><INDENT>fh.write(k + "<STR_LIT:U+002C>" + str(v) + '<STR_LIT:\n>')<EOL><DEDENT><DEDENT><DEDENT>gc.collect()<EOL><DEDENT>return True<EOL>

Take a list of metrics, filter all metrics based on hostname, and metric_type
For each metric, merge the corresponding csv files into one,update corresponding properties such as csv_column_map.
Users can specify functions: raw, count (qps), sum (aggregated value), avg (averaged value)
The timestamp granularity of aggregated submetrics is in seconds (sub-second is not supported)


return True<EOL>

Merge multiple hosts' csv into one csv file. This approach has the benefit of reusing calculate_stats(), but with the penalty of reading the single csv
later for calculate_stats(). However, since file cache will cache the newly written csv files, reading the csv file will not likely be a IO bottleneck.


with open(self.infile, '<STR_LIT:r>') as infh:<EOL><INDENT>max_row_quot = <NUM_LIT:0><EOL>valrow = -<NUM_LIT:1><EOL>thisrowcolumns = {}<EOL>data = {}<EOL>while True:<EOL><INDENT>line1 = infh.readline()<EOL>words = line1.split()<EOL>if words[<NUM_LIT:1>] == "<STR_LIT>" and self.metric_type == "<STR_LIT>":<EOL><INDENT>while True:<EOL><INDENT>line1 = infh.readline()<EOL>words = line1.split()<EOL>if naarad.utils.is_number(words[<NUM_LIT:1>]):<EOL><INDENT>line1 = infh.readline()<EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT><DEDENT>if words[<NUM_LIT:1>] == "<STR_LIT>" and self.metric_type == "<STR_LIT>":<EOL><INDENT>break<EOL><DEDENT>infh.readline()<EOL>last_ts = words[<NUM_LIT:0>].strip().replace('<STR_LIT:T>', '<STR_LIT:U+0020>')<EOL>if not naarad.utils.is_number(words[<NUM_LIT:1>]):<EOL><INDENT>thisrowcolumns[max_row_quot] = words[<NUM_LIT:1>:]<EOL>for column in words[<NUM_LIT:1>:]:<EOL><INDENT>if self.options and column not in self.options:<EOL><INDENT>continue<EOL><DEDENT>data[column] = []<EOL><DEDENT>if self.metric_type == "<STR_LIT>":<EOL><INDENT>data["<STR_LIT>"] = []<EOL><DEDENT>max_row_quot += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>for line in infh:<EOL><INDENT>l = line.strip().split('<STR_LIT:U+0020>', <NUM_LIT:1>)<EOL>if len(l) <= <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>ts = l[<NUM_LIT:0>].strip().replace('<STR_LIT:T>', '<STR_LIT:U+0020>')<EOL>if not ts == last_ts:<EOL><INDENT>last_ts = ts<EOL>valrow = -<NUM_LIT:1><EOL><DEDENT>try:<EOL><INDENT>words = l[<NUM_LIT:1>].strip().split('<STR_LIT:\t>')<EOL><DEDENT>except IndexError:<EOL><INDENT>logger.warn("<STR_LIT>", line)<EOL>continue<EOL><DEDENT>if words[<NUM_LIT:0>] == "<STR_LIT>" or (naarad.utils.is_number(words[<NUM_LIT:0>]) and "<STR_LIT>" in words[<NUM_LIT:1>]):<EOL><INDENT>continue<EOL><DEDENT>if naarad.utils.is_number(words[<NUM_LIT:0>]):<EOL><INDENT>valrow += <NUM_LIT:1><EOL>quot = valrow % max_row_quot<EOL>if quot >= len(thisrowcolumns):<EOL><INDENT>continue<EOL><DEDENT>columns = thisrowcolumns[quot]<EOL>if len(words) > len(columns):<EOL><INDENT>continue<EOL><DEDENT>for i in range(len(words)):<EOL><INDENT>if self.options and columns[i] not in self.options:<EOL><INDENT>continue<EOL><DEDENT>column = columns[i]<EOL>if words[i] == "<STR_LIT>":<EOL><INDENT>words[i] = "<STR_LIT:0>"<EOL><DEDENT>ts = naarad.utils.reconcile_timezones(ts, self.timezone, self.graph_timezone)<EOL>if self.metric_type == "<STR_LIT>":<EOL><INDENT>if column == "<STR_LIT>":<EOL><INDENT>log_seq_no = int(words[i])<EOL><DEDENT>elif column == "<STR_LIT>":<EOL><INDENT>check_pt_age = log_seq_no - int(words[i])<EOL>tup = [ts, str(check_pt_age)]<EOL>data["<STR_LIT>"].append(tup)<EOL><DEDENT><DEDENT>tup = [ts, words[i]]<EOL>data[column].append(tup)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for column in data:<EOL><INDENT>csvfile = self.get_csv(column)<EOL>self.csv_files.append(csvfile)<EOL>with open(csvfile, '<STR_LIT:w>') as outfh:<EOL><INDENT>for tup in data[column]:<EOL><INDENT>outfh.write('<STR_LIT:U+002C>'.join(tup))<EOL>outfh.write('<STR_LIT:\n>')<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Generic parsing method for all other modes


with open(self.infile, '<STR_LIT:r>') as infh:<EOL><INDENT>max_row_quot = <NUM_LIT:0><EOL>valrow = -<NUM_LIT:1><EOL>thisrowcolumns = {}<EOL>data = {}<EOL>last_ts = None<EOL>while True:<EOL><INDENT>line1 = infh.readline()<EOL>words = line1.split()<EOL>infh.readline()<EOL>is_header = True<EOL>for word in words:<EOL><INDENT>if naarad.utils.is_number(word):<EOL><INDENT>last_ts = words[<NUM_LIT:0>].strip().replace('<STR_LIT:T>', '<STR_LIT:U+0020>')<EOL>is_header = False<EOL>break  <EOL><DEDENT><DEDENT>if len(words) > <NUM_LIT:2> and is_header:<EOL><INDENT>thisrowcolumns[max_row_quot] = words[<NUM_LIT:2>:]<EOL>for column in thisrowcolumns[max_row_quot]:<EOL><INDENT>data[column] = []<EOL><DEDENT>max_row_quot += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not last_ts:<EOL><INDENT>logger.warn("<STR_LIT>", self.infile)<EOL>return True<EOL><DEDENT>infh.seek(<NUM_LIT:0>)<EOL>is_bad_line = False<EOL>outfilehandlers = {}<EOL>for line in infh:<EOL><INDENT>l = line.strip().split('<STR_LIT:U+0020>', <NUM_LIT:1>)<EOL>if len(l) <= <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>ts = l[<NUM_LIT:0>].strip().replace('<STR_LIT:T>', '<STR_LIT:U+0020>')<EOL>if ts != last_ts:<EOL><INDENT>last_ts = ts<EOL>valrow = -<NUM_LIT:1><EOL><DEDENT>nameval = l[<NUM_LIT:1>].strip().split('<STR_LIT:\t>', <NUM_LIT:1>)<EOL>try:<EOL><INDENT>words = nameval[<NUM_LIT:1>].split('<STR_LIT:\t>')<EOL><DEDENT>except IndexError:<EOL><INDENT>logger.warn("<STR_LIT>", line)<EOL>continue<EOL><DEDENT>valrow += <NUM_LIT:1><EOL>command = nameval[<NUM_LIT:0>]<EOL>if command not in outfilehandlers:<EOL><INDENT>outfilehandlers[command] = {}<EOL><DEDENT>quot = valrow % max_row_quot<EOL>columns = thisrowcolumns[quot]<EOL>for i in range(len(words)):<EOL><INDENT>if len(words) > len(columns):<EOL><INDENT>logger.warn("<STR_LIT>", line)<EOL>logger.warn("<STR_LIT>", len(words), len(columns))<EOL>break<EOL><DEDENT>if words[i] in columns:<EOL><INDENT>logger.warn("<STR_LIT>", line)<EOL>valrow -= <NUM_LIT:1><EOL>break<EOL><DEDENT>if self.options and columns[i] not in self.options:<EOL><INDENT>continue<EOL><DEDENT>if columns[i] not in outfilehandlers[command]:<EOL><INDENT>outfilehandlers[command][columns[i]] = open(self.get_csv_C(command, columns[i]), '<STR_LIT:w>')<EOL>self.csv_files.append(self.get_csv_C(command, columns[i]))<EOL><DEDENT>ts = naarad.utils.reconcile_timezones(ts, self.timezone, self.graph_timezone)<EOL>outfilehandlers[command][columns[i]].write(ts + '<STR_LIT:U+002C>')<EOL>outfilehandlers[command][columns[i]].write(words[i])<EOL>outfilehandlers[command][columns[i]].write('<STR_LIT:\n>')<EOL><DEDENT><DEDENT>for command in outfilehandlers:<EOL><INDENT>for column in outfilehandlers[command]:<EOL><INDENT>outfilehandlers[command][column].close()<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Special parsing method for Innotop "Replication Status" results (innotop --mode M)


def aggregate_count_over_time(self, metric_store, line_data, transaction_list, aggregate_timestamp):

for transaction in transaction_list:<EOL><INDENT>if line_data.get('<STR_LIT:s>') == '<STR_LIT:true>':<EOL><INDENT>all_qps = metric_store['<STR_LIT>']<EOL><DEDENT>else:<EOL><INDENT>all_qps = metric_store['<STR_LIT>']<EOL><DEDENT>qps = all_qps[transaction]<EOL>if aggregate_timestamp in qps:<EOL><INDENT>qps[aggregate_timestamp] += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>qps[aggregate_timestamp] = <NUM_LIT:1><EOL><DEDENT><DEDENT>return None<EOL>

Organize and store the count of data from the log line into the metric store by metric type, transaction, timestamp

:param dict metric_store: The metric store used to store all the parsed jmeter log data
:param dict line_data: dict with the extracted k:v from the log line
:param list transaction_list: list of transaction to be used for storing the metrics from given line
:param string aggregate_timestamp: timestamp used for storing the raw data. This accounts for aggregation time period
:return: None


def aggregate_values_over_time(self, metric_store, line_data, transaction_list, metric_list, aggregate_timestamp):

for metric in metric_list:<EOL><INDENT>for transaction in transaction_list:<EOL><INDENT>metric_data = reduce(defaultdict.__getitem__, [metric, transaction, aggregate_timestamp], metric_store)<EOL>metric_data.append(float(line_data.get(metric)))<EOL><DEDENT><DEDENT>return None<EOL>

Organize and store the data from the log line into the metric store by metric type, transaction, timestamp

:param dict metric_store: The metric store used to store all the parsed jmeter log data
:param dict line_data: dict with the extracted k:v from the log line
:param list transaction_list: list of transaction to be used for storing the metrics from given line
:param list metric_list: list of metrics to extract from the log line
:param string aggregate_timestamp: timestamp used for storing the raw data. This accounts for aggregation time period
:return: None


def average_values_for_plot(self, metric_store, data, averaging_factor):

for metric, transaction_store in metric_store.items():<EOL><INDENT>for transaction, time_store in transaction_store.items():<EOL><INDENT>for time_stamp, metric_data in sorted(time_store.items()):<EOL><INDENT>if metric in ['<STR_LIT:t>', '<STR_LIT>']:<EOL><INDENT>data[self.get_csv(transaction, metric)].append('<STR_LIT:U+002C>'.join([str(time_stamp), str(sum(map(float, metric_data)) / float(len(metric_data)))]))<EOL>if metric == '<STR_LIT>':<EOL><INDENT>metric_store['<STR_LIT>'][transaction][time_stamp] = sum(map(float, metric_data)) / float(averaging_factor * <NUM_LIT> * <NUM_LIT> / <NUM_LIT>)<EOL>data[self.get_csv(transaction, '<STR_LIT>')].append('<STR_LIT:U+002C>'.join([str(time_stamp), str(metric_store['<STR_LIT>'][transaction][time_stamp])]))<EOL><DEDENT><DEDENT>elif metric in ['<STR_LIT>', '<STR_LIT>']:<EOL><INDENT>data[self.get_csv(transaction, metric)].append('<STR_LIT:U+002C>'.join([str(time_stamp), str(metric_data / float(averaging_factor))]))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return None<EOL>

Create the time series for the various metrics, averaged over the aggregation period being used for plots

:param dict metric_store: The metric store used to store all the parsed jmeter log data
:param dict data: Dict with all the metric data to be output to csv
:param float averaging_factor: averaging factor to be used for calculating the average per second metrics
:return: None


stats_to_calculate = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']  <EOL>percentiles_to_calculate = range(<NUM_LIT:5>, <NUM_LIT>, <NUM_LIT:5>)  <EOL>percentiles_to_calculate.append(<NUM_LIT>)<EOL>for transaction in metric_store['<STR_LIT:t>'].keys():<EOL><INDENT>transaction_key = transaction + '<STR_LIT:.>' + '<STR_LIT>'<EOL>self.calculated_stats[transaction_key], self.calculated_percentiles[transaction_key] =naarad.utils.calculate_stats(list(heapq.merge(*metric_store['<STR_LIT:t>'][transaction].values())),<EOL>stats_to_calculate, percentiles_to_calculate)<EOL>self.update_summary_stats(transaction_key)<EOL>transaction_key = transaction + '<STR_LIT:.>' + '<STR_LIT>'<EOL>if len(metric_store['<STR_LIT>'][transaction].values()) > <NUM_LIT:0>:<EOL><INDENT>self.calculated_stats[transaction_key], self.calculated_percentiles[transaction_key] =naarad.utils.calculate_stats(metric_store['<STR_LIT>'][transaction].values(),<EOL>stats_to_calculate, percentiles_to_calculate)<EOL>self.update_summary_stats(transaction_key)<EOL><DEDENT>transaction_key = transaction + '<STR_LIT:.>' + '<STR_LIT>'<EOL>self.calculated_stats[transaction_key], self.calculated_percentiles[transaction_key] =naarad.utils.calculate_stats(list(heapq.merge(*metric_store['<STR_LIT>'][transaction].values())),<EOL>stats_to_calculate, percentiles_to_calculate)<EOL>self.update_summary_stats(transaction_key)<EOL>if '<STR_LIT>' in metric_store.keys() and transaction in metric_store['<STR_LIT>'].keys():<EOL><INDENT>transaction_key = transaction + '<STR_LIT:.>' + '<STR_LIT>'<EOL>self.calculated_stats[transaction_key], self.calculated_percentiles[transaction_key] =naarad.utils.calculate_stats(metric_store['<STR_LIT>'][transaction].values(),<EOL>stats_to_calculate, percentiles_to_calculate)<EOL>self.update_summary_stats(transaction + '<STR_LIT:.>' + '<STR_LIT>')<EOL><DEDENT>transaction_key = transaction + '<STR_LIT:.>' + '<STR_LIT>'<EOL>self.calculated_stats[transaction_key], self.calculated_percentiles[transaction_key] =naarad.utils.calculate_stats(metric_store['<STR_LIT>'][transaction].values(),<EOL>stats_to_calculate, percentiles_to_calculate)<EOL>self.update_summary_stats(transaction_key)<EOL><DEDENT>return None<EOL>

Calculate key statistics for given data and store in the class variables calculated_stats and calculated_percentiles
calculated_stats:
  'mean', 'std', 'median', 'min', 'max'
calculated_percentiles:
  range(5,101,5), 99
:param dict metric_store: The metric store used to store all the parsed jmeter log data
:return: none


file_status = True<EOL>for infile in self.infile_list:<EOL><INDENT>file_status = file_status and naarad.utils.is_valid_file(infile)<EOL>if not file_status:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>status = self.parse_xml_jtl(self.aggregation_granularity)<EOL>gc.collect()<EOL>return status<EOL>

Parse the Jmeter file and calculate key stats

:return: status of the metric parse


data = defaultdict(list)<EOL>processed_data = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))<EOL>for input_file in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', input_file)<EOL>timestamp_format = None<EOL>tree = ElementTree.parse(input_file)<EOL>samples = tree.findall('<STR_LIT>') + tree.findall('<STR_LIT>')<EOL>for sample in samples:<EOL><INDENT>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(sample.get('<STR_LIT>'))<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(sample.get('<STR_LIT>'), timestamp_format)<EOL>if ts == -<NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.reconcile_timezones(ts, self.timezone, self.graph_timezone)<EOL>aggregate_timestamp, averaging_factor = self.get_aggregation_timestamp(ts, granularity)<EOL>self.aggregate_count_over_time(processed_data, sample, [self._sanitize_label(sample.get('<STR_LIT>')), '<STR_LIT>'], aggregate_timestamp)<EOL>self.aggregate_values_over_time(processed_data, sample, [self._sanitize_label(sample.get('<STR_LIT>')), '<STR_LIT>'], ['<STR_LIT:t>', '<STR_LIT>'], aggregate_timestamp)<EOL>logger.info('<STR_LIT>', input_file)<EOL><DEDENT><DEDENT>logger.info('<STR_LIT>')<EOL>self.average_values_for_plot(processed_data, data, averaging_factor)<EOL>logger.info('<STR_LIT>')<EOL>for csv in data.keys():<EOL><INDENT>self.csv_files.append(csv)<EOL>with open(csv, '<STR_LIT:w>') as csvf:<EOL><INDENT>csvf.write('<STR_LIT:\n>'.join(sorted(data[csv])))<EOL><DEDENT><DEDENT>logger.info('<STR_LIT>')<EOL>self.calculate_key_stats(processed_data)<EOL>return True<EOL>

Parse Jmeter workload output in XML format and extract overall and per transaction data and key statistics

:param string granularity: The time period over which to aggregate and average the raw data. Valid values are 'hour', 'minute' or 'second'
:return: status of the metric parse


v1 = '<STR_LIT>'<EOL>v2 = '<STR_LIT>'<EOL>if len(fields) > <NUM_LIT:0>:<EOL><INDENT>v1 = fields[<NUM_LIT:0>]<EOL><DEDENT>if len(fields) > <NUM_LIT:1>:<EOL><INDENT>v2 = fields[<NUM_LIT:1>]<EOL><DEDENT>return v1, v2<EOL>

:param fields: a list which contains either 0,1,or 2 values
:return: a tuple with default values of '';


for con in self.connections:<EOL><INDENT>ends = con.strip().split('<STR_LIT>')  <EOL>ends = filter(None, ends)  <EOL>if len(ends) == <NUM_LIT:0>:<EOL><INDENT>continue<EOL><DEDENT>if len(ends) > <NUM_LIT:0>:<EOL><INDENT>host1, port1 = self._get_tuple(ends[<NUM_LIT:0>].split('<STR_LIT::>'))<EOL><DEDENT>host2 = '<STR_LIT>'<EOL>port2 = '<STR_LIT>'<EOL>if len(ends) > <NUM_LIT:1>:<EOL><INDENT>host2, port2 = self._get_tuple(ends[<NUM_LIT:1>].split('<STR_LIT::>'))<EOL><DEDENT>self.input_connections.append((host1, port1, host2, port2))<EOL><DEDENT>

Given user input of interested connections, it will extract the info and output a list of tuples.
- input can be multiple values, separated by space;
- either host or port is optional
- it may be just one end,
- e.g., "host1<->host2 host3<->  host1:port1<->host2"
:return: None


for proc in self.processes:<EOL><INDENT>ends = proc.split('<STR_LIT:/>')<EOL>pid, name = self._get_tuple(ends)<EOL>self.input_processes.append((pid, name))<EOL><DEDENT>

Given user input of interested processes, it will extract the info and output a list of tuples.
- input can be multiple values, separated by space;
- either pid or process_name is optional
- e.g., "10001/python 10002/java cpp"
:return: None


def _match_host_port(self, host, port, cur_host, cur_port):

<EOL>host_match = False<EOL>if not host:<EOL><INDENT>host_match = True<EOL><DEDENT>elif cur_host.startswith(host):  <EOL><INDENT>host_match = True<EOL><DEDENT>port_match = False<EOL>if not port:<EOL><INDENT>port_match = True<EOL><DEDENT>elif port == cur_port:<EOL><INDENT>port_match = True<EOL><DEDENT>return host_match and port_match<EOL>

Determine whether user-specified (host,port) matches current (cur_host, cur_port)
:param host,port: The user input of (host,port)
:param cur_host, cur_port: The current connection
:return: True or Not


def _match_processes(self, pid, name, cur_process):

cur_pid, cur_name = self._get_tuple(cur_process.split('<STR_LIT:/>'))<EOL>pid_match = False<EOL>if not pid:<EOL><INDENT>pid_match = True<EOL><DEDENT>elif pid == cur_pid:<EOL><INDENT>pid_match = True<EOL><DEDENT>name_match = False<EOL>if not name:<EOL><INDENT>name_match = True<EOL><DEDENT>elif name == cur_name:<EOL><INDENT>name_match = True<EOL><DEDENT>return pid_match and name_match<EOL>

Determine whether user-specified "pid/processes" contain this process
:param pid: The user input of pid
:param name: The user input of process name
:param process: current process info
:return: True or Not; (if both pid/process are given, then both of them need to match)


def _check_connection(self, local_end, remote_end, process):

<EOL>cur_host1, cur_port1 = self._get_tuple(local_end.split('<STR_LIT::>'))<EOL>cur_host2, cur_port2 = self._get_tuple(remote_end.split('<STR_LIT::>'))<EOL>host_port_is_interested = False<EOL>for (host1, port1, host2, port2) in self.input_connections:<EOL><INDENT>if self._match_host_port(host1, port1, cur_host1, cur_port1) and self._match_host_port(host2, port2, cur_host2, cur_port2):<EOL><INDENT>host_port_is_interested = True<EOL>break<EOL><DEDENT>if self._match_host_port(host1, port1, cur_host2, cur_port2) and self._match_host_port(host2, port2, cur_host1, cur_port1):<EOL><INDENT>host_port_is_interested = True<EOL>break<EOL><DEDENT><DEDENT>process_is_interested = False<EOL>for pid, name in self.input_processes:<EOL><INDENT>if self._match_processes(pid, name, process):<EOL><INDENT>process_is_interested = True<EOL>break<EOL><DEDENT><DEDENT>return cur_host1 + '<STR_LIT:_>' + cur_port1, cur_host2 + '<STR_LIT:_>' + cur_port2, host_port_is_interested and process_is_interested<EOL>

Check whether the connection is of interest or not
:param local_end: Local connection end point, e.g., 'host1:port1'
:param remote_end: Remote connection end point, e.g., 'host2:port2'
:param process: Current connection 's process info, e.g., '1234/firefox'
:return: a tuple of (local_end, remote_end, True/False); e.g. ('host1_23232', 'host2_2222', True)


def _add_data_line(self, data, col, value, ts):

if col in self.column_csv_map:<EOL><INDENT>out_csv = self.column_csv_map[col]<EOL><DEDENT>else:<EOL><INDENT>out_csv = self.get_csv(col)   <EOL>data[out_csv] = []<EOL><DEDENT>data[out_csv].append(ts + "<STR_LIT:U+002C>" + value)<EOL>

Append the data point to the dictionary of "data"
:param data: The dictionary containing all data
:param col: The sub-metric name e.g. 'host1_port1.host2_port2.SendQ'
:param value: integer
:param ts: timestamp
:return: None


<EOL>data = {}  <EOL>for infile in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', infile)<EOL>timestamp_format = None<EOL>with open(infile) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>if '<STR_LIT>' not in line:<EOL><INDENT>continue<EOL><DEDENT>words = line.split()<EOL>if len(words) < <NUM_LIT:8> or words[<NUM_LIT:2>] != '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = words[<NUM_LIT:0>] + "<STR_LIT:U+0020>" + words[<NUM_LIT:1>]<EOL>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(ts)<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(ts, timestamp_format)<EOL>if self.ts_out_of_range(ts):<EOL><INDENT>continue<EOL><DEDENT>local_end, remote_end, interested = self._check_connection(words[<NUM_LIT:5>], words[<NUM_LIT:6>], words[<NUM_LIT:8>])<EOL>if interested:<EOL><INDENT>self._add_data_line(data, local_end + '<STR_LIT:.>' + remote_end + '<STR_LIT>', words[<NUM_LIT:3>], ts)<EOL>self._add_data_line(data, local_end + '<STR_LIT:.>' + remote_end + '<STR_LIT>', words[<NUM_LIT:4>], ts)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>for csv in data.keys():<EOL><INDENT>self.csv_files.append(csv)<EOL>with open(csv, '<STR_LIT:w>') as fh:<EOL><INDENT>fh.write('<STR_LIT:\n>'.join(sorted(data[csv])))<EOL><DEDENT><DEDENT>return True<EOL>

Parse the netstat output file
:return: status of the metric parse


file_status = True<EOL>for input_file in self.infile_list:<EOL><INDENT>file_status = file_status and naarad.utils.is_valid_file(input_file)<EOL>if not file_status:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>status = True<EOL>data = {}  <EOL>for input_file in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', input_file)<EOL>timestamp_format = None<EOL>with open(input_file) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>words = line.split()        <EOL>if len(words) < <NUM_LIT:3>:<EOL><INDENT>continue<EOL><DEDENT>ts = words[<NUM_LIT:0>] + "<STR_LIT:U+0020>" + words[<NUM_LIT:1>]<EOL>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(ts)<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(ts, timestamp_format)<EOL>if self.ts_out_of_range(ts):<EOL><INDENT>continue<EOL><DEDENT>col = words[<NUM_LIT:2>].strip('<STR_LIT::>')<EOL>if self.sub_metrics and col not in self.sub_metrics:<EOL><INDENT>continue<EOL><DEDENT>if len(words) > <NUM_LIT:4> and words[<NUM_LIT:4>]:<EOL><INDENT>unit = words[<NUM_LIT:4>]<EOL><DEDENT>else:<EOL><INDENT>unit = '<STR_LIT>'<EOL><DEDENT>self.sub_metric_unit[col] = unit<EOL>if col in self.column_csv_map:<EOL><INDENT>out_csv = self.column_csv_map[col]<EOL><DEDENT>else:<EOL><INDENT>out_csv = self.get_csv(col)   <EOL>data[out_csv] = []<EOL><DEDENT>data[out_csv].append(ts + "<STR_LIT:U+002C>" + words[<NUM_LIT:3>])<EOL><DEDENT><DEDENT><DEDENT>for csv in data.keys():<EOL><INDENT>self.csv_files.append(csv)<EOL>with open(csv, '<STR_LIT:w>') as fh:<EOL><INDENT>fh.write('<STR_LIT:\n>'.join(sorted(data[csv])))<EOL><DEDENT><DEDENT>return status<EOL>

Parse the vmstat file
:return: status of the metric parse


file_status = True<EOL>for input_file in self.infile_list:<EOL><INDENT>file_status = file_status and naarad.utils.is_valid_file(input_file)<EOL><DEDENT>if not file_status:<EOL><INDENT>return False<EOL><DEDENT>status = True<EOL>data = {}  <EOL>for input_file in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', input_file)<EOL>timestamp_format = None<EOL>with open(input_file) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>words = line.split()          <EOL>if len(words) < <NUM_LIT:3>:<EOL><INDENT>continue<EOL><DEDENT>ts = words[<NUM_LIT:0>] + "<STR_LIT:U+0020>" + words[<NUM_LIT:1>]<EOL>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(ts)<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(ts, timestamp_format)<EOL>if self.ts_out_of_range(ts):<EOL><INDENT>continue<EOL><DEDENT>col = words[<NUM_LIT:2>]<EOL>if self.sub_metrics and col not in self.sub_metrics:<EOL><INDENT>continue<EOL><DEDENT>self.sub_metric_unit[col] = '<STR_LIT>'  <EOL>if col in self.column_csv_map:<EOL><INDENT>out_csv = self.column_csv_map[col]<EOL><DEDENT>else:<EOL><INDENT>out_csv = self.get_csv(col)   <EOL>data[out_csv] = []<EOL><DEDENT>data[out_csv].append(ts + "<STR_LIT:U+002C>" + words[<NUM_LIT:3>])<EOL><DEDENT><DEDENT><DEDENT>for csv in data.keys():<EOL><INDENT>self.csv_files.append(csv)<EOL>with open(csv, '<STR_LIT:w>') as fh:<EOL><INDENT>fh.write('<STR_LIT:\n>'.join(data[csv]))<EOL><DEDENT><DEDENT>return status<EOL>


start_time = <NUM_LIT:0><EOL>end_time = <NUM_LIT:0><EOL>launch_time = <NUM_LIT:0><EOL>nus_update_time = <NUM_LIT:0><EOL>for item in native:<EOL><INDENT>if item[CONSTANTS.LIA_TIMING_NAME] == CONSTANTS.LIA_APP_ON_CREATE and item[CONSTANTS.LIA_START] is not None:<EOL><INDENT>start_time = item[CONSTANTS.LIA_START][CONSTANTS.LIA_LONG]<EOL><DEDENT>if item[CONSTANTS.LIA_TIMING_NAME] == CONSTANTS.LIA_NUS_UPDATE:<EOL><INDENT>if item[CONSTANTS.LIA_TIMING_VALUE] is not None:<EOL><INDENT>nus_update_time = item[CONSTANTS.LIA_TIMING_VALUE][CONSTANTS.LIA_LONG]<EOL><DEDENT>if item[CONSTANTS.LIA_START] is not None:<EOL><INDENT>end_time = item[CONSTANTS.LIA_START][CONSTANTS.LIA_LONG]<EOL><DEDENT><DEDENT><DEDENT>if start_time == <NUM_LIT:0> or end_time == <NUM_LIT:0>:<EOL><INDENT>time_stamp = <NUM_LIT:0><EOL>launch_time = <NUM_LIT:0><EOL><DEDENT>else:<EOL><INDENT>time_stamp = start_time<EOL>launch_time = end_time - start_time<EOL><DEDENT>return (time_stamp, launch_time, nus_update_time)<EOL>

get start time stamp, launch time duration, and nus update time duration from JSON object native
:param JSON OBJECT native
:return: LONG event time stamp, LONG launch time, and LONG nus update time


def get_aggregation_timestamp(self, timestamp, granularity='<STR_LIT>'):

if granularity is None or granularity.lower() == '<STR_LIT:none>':<EOL><INDENT>return int(timestamp), <NUM_LIT:1><EOL><DEDENT>elif granularity == '<STR_LIT>':<EOL><INDENT>return (int(timestamp) / (<NUM_LIT> * <NUM_LIT:1000>)) * <NUM_LIT> * <NUM_LIT:1000>, <NUM_LIT><EOL><DEDENT>elif granularity == '<STR_LIT>':<EOL><INDENT>return (int(timestamp) / (<NUM_LIT> * <NUM_LIT:1000>)) * <NUM_LIT> * <NUM_LIT:1000>, <NUM_LIT><EOL><DEDENT>else:<EOL><INDENT>return (int(timestamp) / <NUM_LIT:1000>) * <NUM_LIT:1000>, <NUM_LIT:1><EOL><DEDENT>

Return a timestamp from the raw epoch time based on the granularity preferences passed in.

:param string timestamp: timestamp from the log line
:param string granularity: aggregation granularity used for plots.
:return: string aggregate_timestamp: timestamp used for metrics aggregation in all functions


def aggregate_count_over_time(self, metric_store, groupby_name, aggregate_timestamp):

all_qps = metric_store['<STR_LIT>']<EOL>qps = all_qps[groupby_name]<EOL>if aggregate_timestamp in qps:<EOL><INDENT>qps[aggregate_timestamp] += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>qps[aggregate_timestamp] = <NUM_LIT:1><EOL><DEDENT>return None<EOL>

Organize and store the count of data from the log line into the metric store by columnm, group name, timestamp

:param dict metric_store: The metric store used to store all the parsed the log data
:param string groupby_name: the group name that the log line belongs to
:param string aggregate_timestamp: timestamp used for storing the raw data. This accounts for aggregation time period
:return: None


def aggregate_values_over_time(self, metric_store, data, groupby_name, column_name, aggregate_timestamp):

<EOL>if self.groupby:<EOL><INDENT>metric_data = reduce(defaultdict.__getitem__, [column_name, '<STR_LIT>', aggregate_timestamp], metric_store)<EOL>metric_data.append(float(data))<EOL><DEDENT>metric_data = reduce(defaultdict.__getitem__, [column_name, groupby_name, aggregate_timestamp], metric_store)<EOL>metric_data.append(float(data))<EOL>return None<EOL>

Organize and store the data from the log line into the metric store by metric type, transaction, timestamp

:param dict metric_store: The metric store used to store all the parsed log data
:param string data: column data in the log line
:param string groupby_name: the group that the data belongs to
:param string column_name: the column name of the data
:param string aggregate_timestamp: timestamp used for storing the raw data. This accounts for aggregation time period
:return: None


for column, groups_store in metric_store.items():<EOL><INDENT>for group, time_store in groups_store.items():<EOL><INDENT>for time_stamp, column_data in sorted(time_store.items()):<EOL><INDENT>if column in ['<STR_LIT>']:<EOL><INDENT>if self.groupby:<EOL><INDENT>data[self.get_csv(column, group)].append('<STR_LIT:U+002C>'.join([str(time_stamp), str(column_data / float(averaging_factor))]))<EOL><DEDENT>else:<EOL><INDENT>data[self.get_csv(column)].append('<STR_LIT:U+002C>'.join([str(time_stamp), str(column_data / float(averaging_factor))]))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if self.groupby:<EOL><INDENT>data[self.get_csv(column, group)].append('<STR_LIT:U+002C>'.join([str(time_stamp), str(sum(map(float, column_data)) / float(len(column_data)))]))<EOL><DEDENT>else:<EOL><INDENT>data[self.get_csv(column)].append('<STR_LIT:U+002C>'.join([str(time_stamp), str(sum(map(float, column_data)) / float(len(column_data)))]))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return None<EOL>

Create the time series for the various metrics, averaged over the aggregation period being used for plots

:param dict metric_store: The metric store used to store all the parsed log data
:param dict data: Dict with all the metric data to be output to csv
:param float averaging_factor: averaging factor to be used for calculating the average per second metrics
:return: None


stats_to_calculate = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>']  <EOL>percentiles_to_calculate = range(<NUM_LIT:0>, <NUM_LIT:100>, <NUM_LIT:1>)  <EOL>for column, groups_store in metric_store.items():<EOL><INDENT>for group, time_store in groups_store.items():<EOL><INDENT>data = metric_store[column][group].values()<EOL>if self.groupby:<EOL><INDENT>column_name = group + '<STR_LIT:.>' + column<EOL><DEDENT>else:<EOL><INDENT>column_name = column<EOL><DEDENT>if column.startswith('<STR_LIT>'):<EOL><INDENT>self.calculated_stats[column_name], self.calculated_percentiles[column_name] = naarad.utils.calculate_stats(data, stats_to_calculate, percentiles_to_calculate)<EOL><DEDENT>else:<EOL><INDENT>self.calculated_stats[column_name], self.calculated_percentiles[column_name] = naarad.utils.calculate_stats(list(heapq.merge(*data)), stats_to_calculate,<EOL>percentiles_to_calculate)<EOL><DEDENT>self.update_summary_stats(column_name)<EOL><DEDENT><DEDENT>

Calculate stats such as percentile and mean

:param dict metric_store: The metric store used to store all the parsed log data
:return: None


metric_type = self.metric_type.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>if metric_type in naarad.naarad_imports.metric_classes or metric_type in naarad.naarad_imports.aggregate_metric_classes:<EOL><INDENT>self.calculate_other_metric_stats()<EOL><DEDENT>else:<EOL><INDENT>self.calculate_base_metric_stats()<EOL><DEDENT>

Calculate stats with different function depending on the metric type:
Data is recorded in memory for base metric type, and use calculate_base_metric_stats()
Data is recorded in CSV file for other metric types, and use calculate_other_metric_stats()


if self.groupby:<EOL><INDENT>plot_data = {}<EOL>for out_csv in sorted(self.csv_files, reverse=True):<EOL><INDENT>csv_filename = os.path.basename(out_csv)<EOL>transaction_name = "<STR_LIT:.>".join(csv_filename.split('<STR_LIT:.>')[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL>if transaction_name in self.anomalies.keys():<EOL><INDENT>highlight_regions = self.anomalies[transaction_name]<EOL><DEDENT>else:<EOL><INDENT>highlight_regions = None<EOL><DEDENT>column = csv_filename.split('<STR_LIT:.>')[-<NUM_LIT:2>]<EOL>transaction_name = '<STR_LIT:U+0020>'.join(csv_filename.split('<STR_LIT:.>')[<NUM_LIT:1>:-<NUM_LIT:2>])<EOL>plot = PD(input_csv=out_csv, csv_column=<NUM_LIT:1>, series_name=transaction_name + '<STR_LIT:.>' + column,<EOL>y_label=column + '<STR_LIT>' + self.sub_metric_description[column] + '<STR_LIT:)>', precision=None, graph_height=<NUM_LIT>, graph_width=<NUM_LIT>, graph_type='<STR_LIT>',<EOL>highlight_regions=highlight_regions)<EOL>if transaction_name in plot_data:<EOL><INDENT>plot_data[transaction_name].append(plot)<EOL><DEDENT>else:<EOL><INDENT>plot_data[transaction_name] = [plot]<EOL><DEDENT><DEDENT>for transaction in plot_data:<EOL><INDENT>graphed, div_file = Metric.graphing_modules[graphing_library].graph_data(plot_data[transaction], self.resource_directory, self.resource_path,<EOL>self.label + '<STR_LIT:.>' + transaction)<EOL>if graphed:<EOL><INDENT>self.plot_files.append(div_file)<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>graphed = False<EOL>for out_csv in self.csv_files:<EOL><INDENT>csv_filename = os.path.basename(out_csv)<EOL>transaction_name = "<STR_LIT:.>".join(csv_filename.split('<STR_LIT:.>')[<NUM_LIT:1>:-<NUM_LIT:1>])<EOL>if transaction_name in self.anomalies.keys():<EOL><INDENT>highlight_regions = self.anomalies[transaction_name]<EOL><DEDENT>else:<EOL><INDENT>highlight_regions = None<EOL><DEDENT>column = self.csv_column_map[out_csv]<EOL>column = naarad.utils.sanitize_string(column)<EOL>graph_title = '<STR_LIT:.>'.join(csv_filename.split('<STR_LIT:.>')[<NUM_LIT:0>:-<NUM_LIT:1>])<EOL>if self.sub_metric_description and column in self.sub_metric_description.keys():<EOL><INDENT>graph_title += '<STR_LIT>' + self.sub_metric_description[column] + '<STR_LIT:)>'<EOL><DEDENT>if self.sub_metric_unit and column in self.sub_metric_unit.keys():<EOL><INDENT>plot_data = [PD(input_csv=out_csv, csv_column=<NUM_LIT:1>, series_name=graph_title, y_label=column + '<STR_LIT>' + self.sub_metric_unit[column] + '<STR_LIT:)>',<EOL>precision=None, graph_height=<NUM_LIT>, graph_width=<NUM_LIT>, graph_type='<STR_LIT>', highlight_regions=highlight_regions)]<EOL><DEDENT>else:<EOL><INDENT>plot_data = [PD(input_csv=out_csv, csv_column=<NUM_LIT:1>, series_name=graph_title, y_label=column, precision=None, graph_height=<NUM_LIT>, graph_width=<NUM_LIT>,<EOL>graph_type='<STR_LIT>', highlight_regions=highlight_regions)]<EOL><DEDENT>graphed, div_file = Metric.graphing_modules[graphing_library].graph_data(plot_data, self.resource_directory, self.resource_path, graph_title)<EOL>if graphed:<EOL><INDENT>self.plot_files.append(div_file)<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>


if not self.important_sub_metrics:<EOL><INDENT>return False<EOL><DEDENT>if sub_metric in self.important_sub_metrics:<EOL><INDENT>return True<EOL><DEDENT>items = sub_metric.split('<STR_LIT:.>')<EOL>if items[-<NUM_LIT:1>] in self.important_sub_metrics:<EOL><INDENT>return True<EOL><DEDENT>return False<EOL>

check whether the given sub metric is in important_sub_metrics list


graphed = False<EOL>for percentile_csv in self.percentiles_files:<EOL><INDENT>csv_filename = os.path.basename(percentile_csv)<EOL>column = self.csv_column_map[percentile_csv.replace("<STR_LIT>", "<STR_LIT:.>")]<EOL>if not self.check_important_sub_metrics(column):<EOL><INDENT>continue<EOL><DEDENT>column = naarad.utils.sanitize_string(column)<EOL>graph_title = '<STR_LIT:.>'.join(csv_filename.split('<STR_LIT:.>')[<NUM_LIT:0>:-<NUM_LIT:1>])<EOL>if self.sub_metric_description and column in self.sub_metric_description.keys():<EOL><INDENT>graph_title += '<STR_LIT>' + self.sub_metric_description[column] + '<STR_LIT:)>'<EOL><DEDENT>if self.sub_metric_unit and column in self.sub_metric_unit.keys():<EOL><INDENT>plot_data = [PD(input_csv=percentile_csv, csv_column=<NUM_LIT:1>, series_name=graph_title, x_label='<STR_LIT>',<EOL>y_label=column + '<STR_LIT>' + self.sub_metric_unit[column] + '<STR_LIT:)>', precision=None, graph_height=<NUM_LIT>, graph_width=<NUM_LIT>, graph_type='<STR_LIT>')]<EOL><DEDENT>else:<EOL><INDENT>plot_data = [PD(input_csv=percentile_csv, csv_column=<NUM_LIT:1>, series_name=graph_title, x_label='<STR_LIT>', y_label=column, precision=None,<EOL>graph_height=<NUM_LIT>, graph_width=<NUM_LIT>, graph_type='<STR_LIT>')]<EOL><DEDENT>graphed, div_file = Metric.graphing_modules[graphing_library].graph_data_on_the_same_graph(plot_data, self.resource_directory,<EOL>self.resource_path, graph_title)<EOL>if graphed:<EOL><INDENT>self.plot_files.append(div_file)<EOL><DEDENT><DEDENT>return True<EOL>


logger.info('<STR_LIT>'.format(lib=graphing_library, name=self.label))<EOL>self.plot_cdf(graphing_library)<EOL>self.plot_timeseries(graphing_library)<EOL>return True<EOL>

graph generates two types of graphs
'time': generate a time-series plot for all submetrics (the x-axis is a time series)
'cdf': generate a CDF plot for important submetrics (the x-axis shows percentiles)


if not self.anomaly_detection_metrics or len(self.anomaly_detection_metrics) <= <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>for submetric in self.anomaly_detection_metrics:<EOL><INDENT>csv_file = self.get_csv(submetric)<EOL>if naarad.utils.is_valid_file(csv_file):<EOL><INDENT>detector = anomaly_detector.AnomalyDetector(csv_file)<EOL>anomalies = detector.get_anomalies()<EOL>if len(anomalies) <= <NUM_LIT:0>:<EOL><INDENT>return<EOL><DEDENT>self.anomalies[submetric] = anomalies<EOL>anomaly_csv_file = os.path.join(self.resource_directory, self.label + '<STR_LIT:.>' + submetric + '<STR_LIT>')<EOL>with open(anomaly_csv_file, '<STR_LIT:w>') as FH:<EOL><INDENT>for anomaly in anomalies:<EOL><INDENT>FH.write("<STR_LIT:U+002C>".join([str(anomaly.anomaly_score), str(anomaly.start_timestamp), str(anomaly.end_timestamp), str(anomaly.exact_timestamp)]))<EOL>FH.write('<STR_LIT:\n>')<EOL><DEDENT><DEDENT><DEDENT><DEDENT>

Detect anomalies in the timeseries data for the submetrics specified in the config file. Identified anomalies are
stored in self.anomalies as well as written to .anomalies.csv file to be used by the client charting page. Anomaly
detection uses the luminol library (http://pypi.python.org/pypi/luminol)


cpu = naarad.utils.sanitize_string(cpu)<EOL>if device is None:<EOL><INDENT>outcsv = os.path.join(self.resource_directory, "<STR_LIT>".format(self.label, cpu))<EOL>self.csv_column_map[outcsv] = cpu<EOL><DEDENT>else:<EOL><INDENT>device = naarad.utils.sanitize_string(device)<EOL>outcsv = os.path.join(self.resource_directory, "<STR_LIT>".format(self.label, cpu, device))<EOL>self.csv_column_map[outcsv] = cpu + '<STR_LIT:.>' + device<EOL><DEDENT>return outcsv<EOL>

Returns the CSV file related to the given metric. The metric is determined by the cpu and device.
The cpu is the CPU as in the interrupts file for example CPU12.
The metric is a combination of the CPU and device. The device consists of IRQ #, the irq device ASCII name.

                                  CPU0   CPU1
2014-10-29 00:27:42.15161    59:    29      2    IR-IO-APIC-edge    timer
                              ^      ^      ^          ^              ^
                              |      |      |          |              |
                            IRQ#   Value  Value   IRQ Device       Ascii Name

This would produce a metric CPU0.timer-IRQ59 and CPU1.timer-IRQ59 so one per IRQ per CPU.

:param cpu: The name of the cpu given as CPU#.
:param device: The device name as given by the system. <ASCII name>-IRQ<IRQ #>
:return: The CSV file for the metric.


return '<STR_LIT>' in line<EOL>

Checks to see if the line is a header line.
The header line style is currently:

  2014-10-29 00:28:42.15161        CPU0   CPU1   CPU2   CPU3  ...

:param line: The line of the file to check.
:return: Boolean of true or false of whether line is header line or not.


cpus = []<EOL>for line in infile:  <EOL><INDENT>if not self.is_header_line(line):<EOL><INDENT>continue<EOL><DEDENT>cpu_header = line.split()<EOL>for cpu_h in cpu_header[<NUM_LIT:2>:]:<EOL><INDENT>if not cpu_h.startswith('<STR_LIT>'):<EOL><INDENT>cpus = []  <EOL>break<EOL><DEDENT>else:<EOL><INDENT>cpus.append(cpu_h)<EOL><DEDENT><DEDENT>if len(cpus) > <NUM_LIT:0>:  <EOL><INDENT>break<EOL><DEDENT><DEDENT>return cpus<EOL>

Parses the file and tries to find the header line. The header line has format:

  2014-10-29 00:28:42.15161        CPU0   CPU1   CPU2   CPU3  ...

So should always have CPU# for each core. This function verifies a good header and
returns the list of CPUs that exist from the header.

:param infile: The opened file in read mode to find the header.
:return cpus: A list of the core names so in this example ['CPU0', 'CPU1', ...]