forward_smooth = {}<EOL>backward_smooth = {}<EOL>output = {}<EOL>if self:<EOL><INDENT>pre = self.values[<NUM_LIT:0>]<EOL>next = self.values[-<NUM_LIT:1>]<EOL>for key, value in self.items():<EOL><INDENT>forward_smooth[key] = smoothing_factor * pre + (<NUM_LIT:1> - smoothing_factor) * value<EOL>pre = forward_smooth[key]<EOL><DEDENT>for key, value in reversed(self.items()):<EOL><INDENT>backward_smooth[key] = smoothing_factor * next + (<NUM_LIT:1> - smoothing_factor) * value<EOL>next = backward_smooth[key]<EOL><DEDENT>for key in forward_smooth.keys():<EOL><INDENT>output[key] = (forward_smooth[key] + backward_smooth[key]) / <NUM_LIT:2><EOL><DEDENT><DEDENT>return TimeSeries(output)<EOL>

return a new time series which is a exponential smoothed version of the original data series.
soomth forward once, backward once, and then take the average.

:param float smoothing_factor: smoothing factor
:return: :class:`TimeSeries` object.


return max(self.timestamps) if self.timestamps else None<EOL>

Return the latest timestamp in the time series.


return numpy.asscalar(numpy.min(self.values)) if self.values else default<EOL>

Calculate the minimum value over the time series.

:param default: Value to return as a default should the calculation not be possible.
:return: Float representing the maximum value or `None`.


if isinstance(other, TimeSeries):<EOL><INDENT>aligned, other_aligned = {}, {}<EOL>i, other_i = self.iteritems_silent(), other.iteritems_silent()<EOL>item, other_item = i.next(), other_i.next()<EOL>while item and other_item:<EOL><INDENT>timestamp, value = item<EOL>other_timestamp, other_value = other_item<EOL>if timestamp == other_timestamp:<EOL><INDENT>aligned[timestamp] = value<EOL>other_aligned[other_timestamp] = other_value<EOL>item = i.next()<EOL>other_item = other_i.next()<EOL><DEDENT>elif timestamp < other_timestamp:<EOL><INDENT>aligned[timestamp] = value<EOL>other_aligned[timestamp] = other_value<EOL>item = i.next()<EOL><DEDENT>else:<EOL><INDENT>aligned[other_timestamp] = value<EOL>other_aligned[other_timestamp] = other_value<EOL>other_item = other_i.next()<EOL><DEDENT><DEDENT>while item:<EOL><INDENT>timestamp, value = item<EOL>aligned[timestamp] = value<EOL>other_aligned[timestamp] = other.values[-<NUM_LIT:1>]<EOL>item = i.next()<EOL><DEDENT>while other_item:<EOL><INDENT>other_timestamp, other_value = other_item<EOL>aligned[other_timestamp] = self.values[-<NUM_LIT:1>]<EOL>other_aligned[other_timestamp] = other_value<EOL>other_item = other_i.next()<EOL><DEDENT>return TimeSeries(aligned), TimeSeries(other_aligned)<EOL><DEDENT>

Align two time series so that len(self) == len(other) and self.timstamps == other.timestamps.

:return: :tuple:(`TimeSeries` object(the aligned self), `TimeSeries` object(the aligned other))


def graph_data_on_the_same_graph(list_of_plots, output_directory, resource_path, output_filename):

maximum_yvalue = -float('<STR_LIT>')<EOL>minimum_yvalue = float('<STR_LIT>')<EOL>plots = curate_plot_list(list_of_plots)<EOL>plot_count = len(plots)<EOL>if plot_count == <NUM_LIT:0>:<EOL><INDENT>return False, None<EOL><DEDENT>graph_height, graph_width, graph_title = get_graph_metadata(plots)<EOL>current_plot_count = <NUM_LIT:0><EOL>fig, axis = plt.subplots()<EOL>fig.set_size_inches(graph_width, graph_height)<EOL>if plot_count < <NUM_LIT:2>:<EOL><INDENT>fig.subplots_adjust(left=CONSTANTS.SUBPLOT_LEFT_OFFSET, bottom=CONSTANTS.SUBPLOT_BOTTOM_OFFSET, right=CONSTANTS.SUBPLOT_RIGHT_OFFSET)<EOL><DEDENT>else:<EOL><INDENT>fig.subplots_adjust(left=CONSTANTS.SUBPLOT_LEFT_OFFSET, bottom=CONSTANTS.SUBPLOT_BOTTOM_OFFSET,<EOL>right=CONSTANTS.SUBPLOT_RIGHT_OFFSET - CONSTANTS.Y_AXIS_OFFSET * (plot_count - <NUM_LIT:2>))<EOL><DEDENT>for plot in plots:<EOL><INDENT>current_plot_count += <NUM_LIT:1><EOL>logger.info('<STR_LIT>' + plot.input_csv + '<STR_LIT>' + output_filename + '<STR_LIT>')<EOL>xval, yval = numpy.loadtxt(plot.input_csv, unpack=True, delimiter='<STR_LIT:U+002C>')<EOL>axis.plot(xval, yval, linestyle='<STR_LIT:->', marker=None, color=get_current_color(current_plot_count), label=plot.plot_label)<EOL>axis.legend()<EOL>maximum_yvalue = max(maximum_yvalue, numpy.amax(yval) * (<NUM_LIT:1.0> + CONSTANTS.ZOOM_FACTOR * current_plot_count))<EOL>minimum_yvalue = min(minimum_yvalue, numpy.amin(yval) * (<NUM_LIT:1.0> - CONSTANTS.ZOOM_FACTOR * current_plot_count))<EOL><DEDENT>axis.yaxis.set_ticks_position('<STR_LIT:left>')<EOL>axis.set_xlabel(plots[<NUM_LIT:0>].x_label)<EOL>axis.set_ylabel(plots[<NUM_LIT:0>].y_label, fontsize=CONSTANTS.Y_LABEL_FONTSIZE)<EOL>axis.set_ylim([minimum_yvalue, maximum_yvalue])<EOL>axis.yaxis.grid(True)<EOL>axis.xaxis.grid(True)<EOL>axis.set_title(graph_title)<EOL>plot_file_name = os.path.join(output_directory, output_filename + "<STR_LIT>")<EOL>fig.savefig(plot_file_name)<EOL>plt.close()<EOL>with open(os.path.join(output_directory, output_filename + '<STR_LIT>'), '<STR_LIT:w>') as div_file:<EOL><INDENT>div_file.write('<STR_LIT>' + os.path.basename(plot_file_name).replace("<STR_LIT>", "<STR_LIT>").replace("<STR_LIT>", "<STR_LIT>") + '<STR_LIT>' +<EOL>resource_path + '<STR_LIT:/>' + os.path.basename(plot_file_name) + '<STR_LIT>' + os.path.basename(plot_file_name) +<EOL>'<STR_LIT>' + os.path.basename(plot_file_name) + '<STR_LIT>')<EOL><DEDENT>return True, os.path.join(output_directory, output_filename + '<STR_LIT>')<EOL>

graph_data_on_the_same_graph: put a list of plots on the same graph: currently it supports CDF


def highlight_region(plt, start_x, end_x):

start_x = convert_to_mdate(start_x)<EOL>end_x = convert_to_mdate(end_x)<EOL>plt.axvspan(start_x, end_x, color=CONSTANTS.HIGHLIGHT_COLOR, alpha=CONSTANTS.HIGHLIGHT_ALPHA)<EOL>

Highlight a region on the chart between the specified start and end x-co-ordinates.
param pyplot plt: matplotlibk pyplot which contains the charts to be highlighted
param string start_x : epoch time millis
param string end_x : epoch time millis


report_count = <NUM_LIT:0><EOL>if self.status != '<STR_LIT:OK>':<EOL><INDENT>return False<EOL><DEDENT>diff_stats = set(self.reports[<NUM_LIT:0>].stats) & set(self.reports[<NUM_LIT:1>].stats)<EOL>if diff_stats:<EOL><INDENT>self.reports[<NUM_LIT:0>].stats = diff_stats<EOL>self.reports[<NUM_LIT:1>].stats = diff_stats<EOL><DEDENT>else:<EOL><INDENT>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>')<EOL>return False<EOL><DEDENT>for report in self.reports:<EOL><INDENT>report.label = report_count<EOL>report_count += <NUM_LIT:1><EOL>report.local_location = os.path.join(self.resource_directory, str(report.label))<EOL>try:<EOL><INDENT>os.makedirs(report.local_location)<EOL><DEDENT>except OSError as exeption:<EOL><INDENT>if exeption.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>if report.remote_location != '<STR_LIT>':<EOL><INDENT>naarad.httpdownload.download_url_list(map(lambda x: report.remote_location + '<STR_LIT:/>' + self.resource_path + '<STR_LIT:/>' + x, report.stats), report.local_location)<EOL><DEDENT>else:<EOL><INDENT>for filename in report.stats:<EOL><INDENT>shutil.copy(os.path.join(os.path.join(report.location, self.resource_path), filename), report.local_location)<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Identify what summary stats exist in both the diffed reports and download them to the diff report resources directory
:return: True/False : return status of whether the download of summary stats succeeded.


if not os.path.exists(self.resource_directory):<EOL><INDENT>os.makedirs(self.resource_directory)<EOL><DEDENT>self.copy_local_includes()<EOL>div_html = '<STR_LIT>'<EOL>for plot_div in sorted(self.plot_files):<EOL><INDENT>with open(plot_div, '<STR_LIT:r>') as div_file:<EOL><INDENT>div_html += '<STR_LIT:\n>' + div_file.read()<EOL><DEDENT><DEDENT>template_loader = FileSystemLoader(self.get_resources_location())<EOL>template_environment = Environment(loader=template_loader)<EOL>template_environment.filters['<STR_LIT>'] = naarad.utils.sanitize_string<EOL>diff_html = template_environment.get_template(CONSTANTS.TEMPLATE_HEADER).render(custom_stylesheet_includes=CONSTANTS.STYLESHEET_INCLUDES,<EOL>custom_javascript_includes=CONSTANTS.JAVASCRIPT_INCLUDES,<EOL>resource_path=self.resource_path,<EOL>report_title='<STR_LIT>') + '<STR_LIT:\n>'<EOL>diff_html += template_environment.get_template(CONSTANTS.TEMPLATE_DIFF_PAGE).render(diff_data=self.diff_data, plot_div_content=div_html,<EOL>reports=self.reports, sla_failure_list=self.sla_failure_list,<EOL>sla_map=self.sla_map) + '<STR_LIT:\n>'<EOL>diff_html += template_environment.get_template(CONSTANTS.TEMPLATE_FOOTER).render()<EOL>return diff_html<EOL>

Generate the summary diff report html from template
:return: generated html to be written to disk


diff_datasource = sorted(set(self.reports[<NUM_LIT:0>].datasource) & set(self.reports[<NUM_LIT:1>].datasource))<EOL>graphed = False<EOL>for submetric in diff_datasource:<EOL><INDENT>baseline_csv = naarad.utils.get_default_csv(self.reports[<NUM_LIT:0>].local_location, (submetric + '<STR_LIT>'))<EOL>current_csv = naarad.utils.get_default_csv(self.reports[<NUM_LIT:1>].local_location, (submetric + '<STR_LIT>'))<EOL>if (not (naarad.utils.is_valid_file(baseline_csv) & naarad.utils.is_valid_file(current_csv))):<EOL><INDENT>continue<EOL><DEDENT>baseline_plot = PD(input_csv=baseline_csv, csv_column=<NUM_LIT:1>, series_name=submetric, y_label=submetric, precision=None, graph_height=<NUM_LIT>, graph_width=<NUM_LIT>,<EOL>graph_type='<STR_LIT>', plot_label='<STR_LIT>', x_label='<STR_LIT>')<EOL>current_plot = PD(input_csv=current_csv, csv_column=<NUM_LIT:1>, series_name=submetric, y_label=submetric, precision=None, graph_height=<NUM_LIT>, graph_width=<NUM_LIT>,<EOL>graph_type='<STR_LIT>', plot_label='<STR_LIT>', x_label='<STR_LIT>')<EOL>graphed, div_file = Diff.graphing_modules[graphing_library].graph_data_on_the_same_graph([baseline_plot, current_plot],<EOL>os.path.join(self.output_directory, self.resource_path),<EOL>self.resource_path, (submetric + '<STR_LIT>'))<EOL>if graphed:<EOL><INDENT>self.plot_files.append(div_file)<EOL><DEDENT><DEDENT>return True<EOL>

Generate CDF diff plots of the submetrics


resource_folder = self.get_resources_location()<EOL>for stylesheet in self.stylesheet_includes:<EOL><INDENT>if ('<STR_LIT:http>' not in stylesheet) and naarad.utils.is_valid_file(os.path.join(resource_folder, stylesheet)):<EOL><INDENT>shutil.copy(os.path.join(resource_folder, stylesheet), self.resource_directory)<EOL><DEDENT><DEDENT>for javascript in self.javascript_includes:<EOL><INDENT>if ('<STR_LIT:http>' not in javascript) and naarad.utils.is_valid_file(os.path.join(resource_folder, javascript)):<EOL><INDENT>shutil.copy(os.path.join(resource_folder, javascript), self.resource_directory)<EOL><DEDENT><DEDENT>return None<EOL>

Copy local js/css includes from naarad resources to the report/resources directory
:return: None


if not os.path.exists(self.resource_directory):<EOL><INDENT>os.makedirs(self.resource_directory)<EOL><DEDENT>self.copy_local_includes()<EOL>template_loader = FileSystemLoader(self.get_resources_location())<EOL>template_environment = Environment(loader=template_loader)<EOL>client_html = template_environment.get_template(CONSTANTS.TEMPLATE_HEADER).render(custom_stylesheet_includes=CONSTANTS.STYLESHEET_INCLUDES,<EOL>custom_javascript_includes=CONSTANTS.JAVASCRIPT_INCLUDES,<EOL>resource_path=self.resource_path,<EOL>report_title='<STR_LIT>') + '<STR_LIT:\n>'<EOL>client_html += template_environment.get_template(CONSTANTS.TEMPLATE_DIFF_CLIENT_CHARTING).render(data_series=data_sources,<EOL>resource_path=self.resource_path) + '<STR_LIT:\n>'<EOL>client_html += template_environment.get_template(CONSTANTS.TEMPLATE_FOOTER).render()<EOL>return client_html<EOL>

Create the client charting page for the diff report, with time series data from the two diffed reports.
:return: generated html to be written to disk


for report in self.reports:<EOL><INDENT>if report.remote_location == '<STR_LIT>':<EOL><INDENT>if naarad.utils.is_valid_file(os.path.join(os.path.join(report.location, self.resource_path), metafile)):<EOL><INDENT>with open(os.path.join(os.path.join(report.location, self.resource_path), metafile), '<STR_LIT:r>') as meta_file:<EOL><INDENT>if metafile == CONSTANTS.STATS_CSV_LIST_FILE:<EOL><INDENT>report.stats = meta_file.readlines()[<NUM_LIT:0>].split('<STR_LIT:U+002C>')<EOL><DEDENT>elif metafile == CONSTANTS.PLOTS_CSV_LIST_FILE:<EOL><INDENT>report.datasource = meta_file.readlines()[<NUM_LIT:0>].split('<STR_LIT:U+002C>')<EOL><DEDENT>elif metafile == CONSTANTS.CDF_PLOTS_CSV_LIST_FILE:<EOL><INDENT>report.cdf_datasource = meta_file.readlines()[<NUM_LIT:0>].split('<STR_LIT:U+002C>')<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>report.status = '<STR_LIT>'<EOL>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>', report.label)<EOL>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>stats_url = report.remote_location + '<STR_LIT:/>' + self.resource_path + '<STR_LIT:/>' + metafile<EOL>meta_file_data = naarad.httpdownload.stream_url(stats_url)<EOL>if meta_file_data:<EOL><INDENT>if metafile == CONSTANTS.STATS_CSV_LIST_FILE:<EOL><INDENT>report.stats = meta_file_data.split('<STR_LIT:U+002C>')<EOL><DEDENT>elif metafile == CONSTANTS.PLOTS_CSV_LIST_FILE:<EOL><INDENT>report.datasource = meta_file_data.split('<STR_LIT:U+002C>')<EOL><DEDENT>elif metafile == CONSTANTS.CDF_PLOTS_CSV_LIST_FILE:<EOL><INDENT>report.cdf_datasource = meta_file_data.split('<STR_LIT:U+002C>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>report.status = '<STR_LIT>'<EOL>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>', report.label)<EOL>return False<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Determine what summary stats, time series, and CDF csv exist for the reports that need to be diffed.
:return: boolean: return whether the summary stats / time series / CDF csv summary was successfully located


report_count = <NUM_LIT:0><EOL>if self.status != '<STR_LIT:OK>':<EOL><INDENT>return False<EOL><DEDENT>diff_cdf_datasource = sorted(set(self.reports[<NUM_LIT:0>].cdf_datasource) & set(self.reports[<NUM_LIT:1>].cdf_datasource))<EOL>if diff_cdf_datasource:<EOL><INDENT>self.reports[<NUM_LIT:0>].cdf_datasource = diff_cdf_datasource<EOL>self.reports[<NUM_LIT:1>].cdf_datasource = diff_cdf_datasource<EOL><DEDENT>else:<EOL><INDENT>self.status = '<STR_LIT>'<EOL>logger.error('<STR_LIT>')<EOL>return False<EOL><DEDENT>for report in self.reports:<EOL><INDENT>report.label = report_count<EOL>report_count += <NUM_LIT:1><EOL>report.local_location = os.path.join(self.resource_directory, str(report.label))<EOL>try:<EOL><INDENT>os.makedirs(report.local_location)<EOL><DEDENT>except OSError as exeption:<EOL><INDENT>if exeption.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>if report.remote_location != '<STR_LIT>':<EOL><INDENT>naarad.httpdownload.download_url_list(map(lambda x: report.remote_location + '<STR_LIT:/>' + self.resource_path + '<STR_LIT:/>' + x + '<STR_LIT>', report.cdf_datasource),<EOL>report.local_location)<EOL><DEDENT>else:<EOL><INDENT>for filename in report.cdf_datasource:<EOL><INDENT>try:<EOL><INDENT>shutil.copy(os.path.join(os.path.join(report.location, self.resource_path), filename + '<STR_LIT>'), report.local_location)<EOL><DEDENT>except IOError as exeption:<EOL><INDENT>continue<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return True<EOL>

Identify what cdf series exist in both the diffed reports and download them to the diff report resources directory
:return: True/False : return status of whether the download of time series resources succeeded.


words = words[<NUM_LIT:1>:]<EOL>length = len(words) / <NUM_LIT:2>  <EOL>values = {}<EOL>for offset in range(length):<EOL><INDENT>k = words[<NUM_LIT:2> * offset + <NUM_LIT:1>].strip('<STR_LIT:U+002C>')<EOL>v = words[<NUM_LIT:2> * offset]<EOL>values['<STR_LIT>' + k] = v<EOL><DEDENT>self.put_values_into_data(values)<EOL>

Process the line starting with "Tasks:"
Example log:   Tasks: 446 total,   1 running, 442 sleeping,   2 stopped,   1 zombie


with open(self.infile, '<STR_LIT:r>') as infh:<EOL><INDENT>max_row_quot = <NUM_LIT:0><EOL>valrow = -<NUM_LIT:1><EOL>thisrowcolumns = {}<EOL>data = {}<EOL>last_ts = None<EOL>while True:<EOL><INDENT>line1 = infh.readline()<EOL>words = line1.split()<EOL>infh.readline()<EOL>is_header = True<EOL>for word in words:<EOL><INDENT>if naarad.utils.is_number(word):<EOL><INDENT>last_ts = words[<NUM_LIT:0>].strip().replace('<STR_LIT:T>', '<STR_LIT:U+0020>')<EOL>is_header = False<EOL>break  <EOL><DEDENT><DEDENT>if len(words) > <NUM_LIT:2> and is_header:<EOL><INDENT>thisrowcolumns[max_row_quot] = words[<NUM_LIT:2>:]<EOL>for column in thisrowcolumns[max_row_quot]:<EOL><INDENT>data[column] = []<EOL><DEDENT>max_row_quot += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if not last_ts:<EOL><INDENT>logger.warn("<STR_LIT>", self.infile)<EOL>return True<EOL><DEDENT>infh.seek(<NUM_LIT:0>)<EOL>is_bad_line = False<EOL>outfilehandlers = {}<EOL>for line in infh:<EOL><INDENT>l = line.strip().split('<STR_LIT:U+0020>', <NUM_LIT:1>)<EOL>if len(l) <= <NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>ts = l[<NUM_LIT:0>].strip().replace('<STR_LIT:T>', '<STR_LIT:U+0020>')<EOL>if ts != last_ts:<EOL><INDENT>last_ts = ts<EOL>valrow = -<NUM_LIT:1><EOL><DEDENT>nameval = l[<NUM_LIT:1>].strip().split('<STR_LIT:\t>', <NUM_LIT:1>)<EOL>try:<EOL><INDENT>words = nameval[<NUM_LIT:1>].split('<STR_LIT:\t>')<EOL><DEDENT>except IndexError:<EOL><INDENT>logger.warn("<STR_LIT>", line)<EOL>continue<EOL><DEDENT>valrow += <NUM_LIT:1><EOL>command = nameval[<NUM_LIT:0>]<EOL>if command not in outfilehandlers:<EOL><INDENT>outfilehandlers[command] = {}<EOL><DEDENT>quot = valrow % max_row_quot<EOL>columns = thisrowcolumns[quot]<EOL>for i in range(len(words)):<EOL><INDENT>if len(words) > len(columns):<EOL><INDENT>logger.warn("<STR_LIT>", line)<EOL>logger.warn("<STR_LIT>", len(words), len(columns))<EOL>break<EOL><DEDENT>if words[i] in columns:<EOL><INDENT>logger.warn("<STR_LIT>", line)<EOL>valrow -= <NUM_LIT:1><EOL>break<EOL><DEDENT>if self.options and columns[i] not in self.options:<EOL><INDENT>continue<EOL><DEDENT>if columns[i] not in outfilehandlers[command]:<EOL><INDENT>outfilehandlers[command][columns[i]] = open(self.get_csv_C(command, columns[i]), '<STR_LIT:w>')<EOL>self.csv_files.append(self.get_csv_C(command, columns[i]))<EOL><DEDENT>ts = naarad.utils.reconcile_timezones(ts, self.timezone, self.graph_timezone)<EOL>outfilehandlers[command][columns[i]].write(ts + '<STR_LIT:U+002C>')<EOL>outfilehandlers[command][columns[i]].write(words[i])<EOL>outfilehandlers[command][columns[i]].write('<STR_LIT:\n>')<EOL><DEDENT><DEDENT>for command in outfilehandlers:<EOL><INDENT>for column in outfilehandlers[command]:<EOL><INDENT>outfilehandlers[command][column].close()<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Special parsing method for Innotop "Replication Status" results (innotop --mode M)


file_status = True<EOL>for infile in self.infile_list:<EOL><INDENT>file_status = file_status and naarad.utils.is_valid_file(infile)<EOL>if not file_status:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>status = self.parse_xml_jtl(self.aggregation_granularity)<EOL>gc.collect()<EOL>return status<EOL>

Parse the Jmeter file and calculate key stats

:return: status of the metric parse


data = defaultdict(list)<EOL>processed_data = defaultdict(lambda: defaultdict(lambda: defaultdict(list)))<EOL>for input_file in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', input_file)<EOL>timestamp_format = None<EOL>tree = ElementTree.parse(input_file)<EOL>samples = tree.findall('<STR_LIT>') + tree.findall('<STR_LIT>')<EOL>for sample in samples:<EOL><INDENT>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(sample.get('<STR_LIT>'))<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(sample.get('<STR_LIT>'), timestamp_format)<EOL>if ts == -<NUM_LIT:1>:<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.reconcile_timezones(ts, self.timezone, self.graph_timezone)<EOL>aggregate_timestamp, averaging_factor = self.get_aggregation_timestamp(ts, granularity)<EOL>self.aggregate_count_over_time(processed_data, sample, [self._sanitize_label(sample.get('<STR_LIT>')), '<STR_LIT>'], aggregate_timestamp)<EOL>self.aggregate_values_over_time(processed_data, sample, [self._sanitize_label(sample.get('<STR_LIT>')), '<STR_LIT>'], ['<STR_LIT:t>', '<STR_LIT>'], aggregate_timestamp)<EOL>logger.info('<STR_LIT>', input_file)<EOL><DEDENT><DEDENT>logger.info('<STR_LIT>')<EOL>self.average_values_for_plot(processed_data, data, averaging_factor)<EOL>logger.info('<STR_LIT>')<EOL>for csv in data.keys():<EOL><INDENT>self.csv_files.append(csv)<EOL>with open(csv, '<STR_LIT:w>') as csvf:<EOL><INDENT>csvf.write('<STR_LIT:\n>'.join(sorted(data[csv])))<EOL><DEDENT><DEDENT>logger.info('<STR_LIT>')<EOL>self.calculate_key_stats(processed_data)<EOL>return True<EOL>

Parse Jmeter workload output in XML format and extract overall and per transaction data and key statistics

:param string granularity: The time period over which to aggregate and average the raw data. Valid values are 'hour', 'minute' or 'second'
:return: status of the metric parse


def aggregate_values_over_time(self, metric_store, line_data, transaction_list, metric_list, aggregate_timestamp):

for metric in metric_list:<EOL><INDENT>for transaction in transaction_list:<EOL><INDENT>metric_data = reduce(defaultdict.__getitem__, [metric, transaction, aggregate_timestamp], metric_store)<EOL>metric_data.append(float(line_data.get(metric)))<EOL><DEDENT><DEDENT>return None<EOL>

Organize and store the data from the log line into the metric store by metric type, transaction, timestamp

:param dict metric_store: The metric store used to store all the parsed jmeter log data
:param dict line_data: dict with the extracted k:v from the log line
:param list transaction_list: list of transaction to be used for storing the metrics from given line
:param list metric_list: list of metrics to extract from the log line
:param string aggregate_timestamp: timestamp used for storing the raw data. This accounts for aggregation time period
:return: None


v1 = '<STR_LIT>'<EOL>v2 = '<STR_LIT>'<EOL>if len(fields) > <NUM_LIT:0>:<EOL><INDENT>v1 = fields[<NUM_LIT:0>]<EOL><DEDENT>if len(fields) > <NUM_LIT:1>:<EOL><INDENT>v2 = fields[<NUM_LIT:1>]<EOL><DEDENT>return v1, v2<EOL>

:param fields: a list which contains either 0,1,or 2 values
:return: a tuple with default values of '';


def _add_data_line(self, data, col, value, ts):

if col in self.column_csv_map:<EOL><INDENT>out_csv = self.column_csv_map[col]<EOL><DEDENT>else:<EOL><INDENT>out_csv = self.get_csv(col)   <EOL>data[out_csv] = []<EOL><DEDENT>data[out_csv].append(ts + "<STR_LIT:U+002C>" + value)<EOL>

Append the data point to the dictionary of "data"
:param data: The dictionary containing all data
:param col: The sub-metric name e.g. 'host1_port1.host2_port2.SendQ'
:param value: integer
:param ts: timestamp
:return: None


def _match_host_port(self, host, port, cur_host, cur_port):

<EOL>host_match = False<EOL>if not host:<EOL><INDENT>host_match = True<EOL><DEDENT>elif cur_host.startswith(host):  <EOL><INDENT>host_match = True<EOL><DEDENT>port_match = False<EOL>if not port:<EOL><INDENT>port_match = True<EOL><DEDENT>elif port == cur_port:<EOL><INDENT>port_match = True<EOL><DEDENT>return host_match and port_match<EOL>

Determine whether user-specified (host,port) matches current (cur_host, cur_port)
:param host,port: The user input of (host,port)
:param cur_host, cur_port: The current connection
:return: True or Not


def aggregate_count_over_time(self, metric_store, groupby_name, aggregate_timestamp):

all_qps = metric_store['<STR_LIT>']<EOL>qps = all_qps[groupby_name]<EOL>if aggregate_timestamp in qps:<EOL><INDENT>qps[aggregate_timestamp] += <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>qps[aggregate_timestamp] = <NUM_LIT:1><EOL><DEDENT>return None<EOL>

Organize and store the count of data from the log line into the metric store by columnm, group name, timestamp

:param dict metric_store: The metric store used to store all the parsed the log data
:param string groupby_name: the group name that the log line belongs to
:param string aggregate_timestamp: timestamp used for storing the raw data. This accounts for aggregation time period
:return: None


metric_type = self.metric_type.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>if metric_type in naarad.naarad_imports.metric_classes or metric_type in naarad.naarad_imports.aggregate_metric_classes:<EOL><INDENT>self.calculate_other_metric_stats()<EOL><DEDENT>else:<EOL><INDENT>self.calculate_base_metric_stats()<EOL><DEDENT>

Calculate stats with different function depending on the metric type:
Data is recorded in memory for base metric type, and use calculate_base_metric_stats()
Data is recorded in CSV file for other metric types, and use calculate_other_metric_stats()


cpus = []<EOL>for line in infile:  <EOL><INDENT>if not self.is_header_line(line):<EOL><INDENT>continue<EOL><DEDENT>cpu_header = line.split()<EOL>for cpu_h in cpu_header[<NUM_LIT:2>:]:<EOL><INDENT>if not cpu_h.startswith('<STR_LIT>'):<EOL><INDENT>cpus = []  <EOL>break<EOL><DEDENT>else:<EOL><INDENT>cpus.append(cpu_h)<EOL><DEDENT><DEDENT>if len(cpus) > <NUM_LIT:0>:  <EOL><INDENT>break<EOL><DEDENT><DEDENT>return cpus<EOL>

Parses the file and tries to find the header line. The header line has format:

  2014-10-29 00:28:42.15161        CPU0   CPU1   CPU2   CPU3  ...

So should always have CPU# for each core. This function verifies a good header and
returns the list of CPUs that exist from the header.

:param infile: The opened file in read mode to find the header.
:return cpus: A list of the core names so in this example ['CPU0', 'CPU1', ...]


file_status = True<EOL>for input_file in self.infile_list:<EOL><INDENT>file_status = file_status and naarad.utils.is_valid_file(input_file)<EOL>if not file_status:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>status = True<EOL>cur_zone = None<EOL>cur_submetric = None<EOL>cur_value = None<EOL>data = {}  <EOL>for input_file in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', input_file)<EOL>timestamp_format = None<EOL>with open(input_file) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>words = line.replace('<STR_LIT:U+002C>', '<STR_LIT:U+0020>').split()           <EOL>if len(words) < <NUM_LIT:3>:<EOL><INDENT>continue<EOL><DEDENT>ts = words[<NUM_LIT:0>] + "<STR_LIT:U+0020>" + words[<NUM_LIT:1>]<EOL>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(ts)<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(ts, timestamp_format)<EOL>if self.ts_out_of_range(ts):<EOL><INDENT>continue<EOL><DEDENT>if words[<NUM_LIT:2>] == '<STR_LIT>':  <EOL><INDENT>cols = words[<NUM_LIT:2>:]<EOL>cur_zone = '<STR_LIT:.>'.join(cols)<EOL>continue<EOL><DEDENT>elif words[<NUM_LIT:2>] == '<STR_LIT>':  <EOL><INDENT>cur_submetric = words[<NUM_LIT:2>] + '<STR_LIT:.>' + words[<NUM_LIT:3>]  <EOL>cur_value = words[<NUM_LIT:4>]<EOL><DEDENT>elif words[<NUM_LIT:2>] in self.processed_sub_metrics:<EOL><INDENT>cur_submetric = '<STR_LIT>' + '<STR_LIT:.>' + words[<NUM_LIT:2>]  <EOL>cur_value = words[<NUM_LIT:3>]<EOL><DEDENT>elif words[<NUM_LIT:2>] in self.skipped_sub_metrics:<EOL><INDENT>continue<EOL><DEDENT>else:   <EOL><INDENT>cur_submetric = words[<NUM_LIT:2>]<EOL>cur_value = words[<NUM_LIT:3>]<EOL><DEDENT>col = cur_zone + '<STR_LIT:.>' + cur_submetric  <EOL>if cur_zone and self.zones and cur_zone not in self.zones:<EOL><INDENT>continue<EOL><DEDENT>self.sub_metric_unit[col] = '<STR_LIT>'  <EOL>if self.sub_metrics and cur_submetric and cur_submetric not in self.sub_metrics:<EOL><INDENT>continue<EOL><DEDENT>if col in self.column_csv_map:<EOL><INDENT>out_csv = self.column_csv_map[col]<EOL><DEDENT>else:<EOL><INDENT>out_csv = self.get_csv(col)   <EOL>data[out_csv] = []<EOL><DEDENT>data[out_csv].append(ts + "<STR_LIT:U+002C>" + cur_value)<EOL><DEDENT><DEDENT><DEDENT>for csv in data.keys():<EOL><INDENT>self.csv_files.append(csv)<EOL>with open(csv, '<STR_LIT:w>') as fh:<EOL><INDENT>fh.write('<STR_LIT:\n>'.join(sorted(data[csv])))<EOL><DEDENT><DEDENT>return status<EOL>

Parse the vmstat file
:return: status of the metric parse


def analyze(self, input_directory, output_directory, **kwargs):

is_api_call = True<EOL>if len(self._analyses) == <NUM_LIT:0>:<EOL><INDENT>if '<STR_LIT>' not in kwargs.keys():<EOL><INDENT>return CONSTANTS.ERROR<EOL><DEDENT>self.create_analysis(kwargs['<STR_LIT>'])<EOL><DEDENT>if '<STR_LIT:args>' in kwargs:<EOL><INDENT>self._process_args(self._analyses[<NUM_LIT:0>], kwargs['<STR_LIT:args>'])<EOL>is_api_call = False<EOL><DEDENT>error_count = <NUM_LIT:0><EOL>self._input_directory = input_directory<EOL>self._output_directory = output_directory<EOL>for test_id in sorted(self._analyses.keys()):<EOL><INDENT>if not self._analyses[test_id].input_directory:<EOL><INDENT>self._analyses[test_id].input_directory = input_directory<EOL><DEDENT>if not self._analyses[test_id].output_directory:<EOL><INDENT>if len(self._analyses) > <NUM_LIT:1>:<EOL><INDENT>self._analyses[test_id].output_directory = os.path.join(output_directory, str(test_id))<EOL><DEDENT>else:<EOL><INDENT>self._analyses[test_id].output_directory = output_directory<EOL><DEDENT><DEDENT>if('<STR_LIT>' in kwargs.keys()) and (not self._analyses[test_id].config):<EOL><INDENT>self._analyses[test_id].config = kwargs['<STR_LIT>']<EOL><DEDENT>self._create_output_directories(self._analyses[test_id])<EOL>self._analyses[test_id].status = self.run(self._analyses[test_id], is_api_call, **kwargs)<EOL>if self._analyses[test_id].status != CONSTANTS.OK:<EOL><INDENT>error_count += <NUM_LIT:1><EOL><DEDENT><DEDENT>if len(self._analyses) == <NUM_LIT:1>:<EOL><INDENT>return self._analyses[<NUM_LIT:0>].status<EOL><DEDENT>elif error_count > <NUM_LIT:0>:<EOL><INDENT>return CONSTANTS.ERROR<EOL><DEDENT>else:<EOL><INDENT>return CONSTANTS.OK<EOL><DEDENT>

Run all the analysis saved in self._analyses, sorted by test_id.
This is useful when Naarad() is used by other programs and multiple analyses are run
In naarad CLI mode, len(_analyses) == 1
:param: input_directory: location of log files
:param: output_directory: root directory for analysis output
:param: **kwargs: Optional keyword args
:return: int: status code.


failed_analyses = []<EOL>for test_id in self._analyses.keys():<EOL><INDENT>if self._analyses[test_id].status != CONSTANTS.OK:<EOL><INDENT>failed_analyses.append(test_id)<EOL><DEDENT><DEDENT>return failed_analyses<EOL>

Returns a list of test_id for which naarad analysis failed
:return: list of test_ids


self._default_test_id += <NUM_LIT:1><EOL>self._analyses[self._default_test_id] = _Analysis(ts_start=None, config=config, test_id=self._default_test_id)<EOL>

Create Analysis and save in Naarad from config
:param config:
:return:


metric_list = []<EOL>log_files = os.listdir(input_directory)<EOL>for log_file in log_files:<EOL><INDENT>if log_file in CONSTANTS.SUPPORTED_FILENAME_MAPPING.keys():<EOL><INDENT>metric_list.append(initialize_metric(CONSTANTS.SUPPORTED_FILENAME_MAPPING[log_file], [log_file], None, [], output_directory, CONSTANTS.RESOURCE_PATH,<EOL>CONSTANTS.SUPPORTED_FILENAME_MAPPING[log_file], None, None, {}, None, None, {}))<EOL><DEDENT>else:<EOL><INDENT>logger.warning('<STR_LIT>', log_file)<EOL><DEDENT><DEDENT>return metric_list<EOL>

Auto discover metric types from the files that exist in input_directory and return a list of metrics
:param: input_directory: The location to scan for log files
:param: output_directory: The location for the report


kill_after_seconds = None<EOL>try:<EOL><INDENT>run_cmd = config_obj.get(section, '<STR_LIT>')<EOL>run_rank = int(config_obj.get(section, '<STR_LIT>'))<EOL><DEDENT>except ConfigParser.NoOptionError:<EOL><INDENT>logger.exception("<STR_LIT>" + section)<EOL>sys.exit()<EOL><DEDENT>except ValueError:<EOL><INDENT>logger.error("<STR_LIT>", config_obj.get(section, '<STR_LIT>'), section)<EOL>sys.exit()<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>run_type = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>run_type = CONSTANTS.RUN_TYPE_WORKLOAD<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>run_order = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>run_order = CONSTANTS.PRE_ANALYSIS_RUN<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>call_type = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>call_type = '<STR_LIT>'<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>kill_after_seconds = int(config_obj.get(section, '<STR_LIT>'))<EOL><DEDENT>except ValueError:<EOL><INDENT>logger.error("<STR_LIT>", config_obj.get(section, '<STR_LIT>'), section)<EOL><DEDENT><DEDENT>if call_type == '<STR_LIT>':<EOL><INDENT>run_step_obj = Local_Cmd(run_type, run_cmd, call_type, run_order, run_rank, kill_after_seconds=kill_after_seconds)<EOL><DEDENT>else:<EOL><INDENT>logger.error('<STR_LIT>')<EOL>run_step_obj = None<EOL><DEDENT>return run_step_obj<EOL>

Parse a RUN-STEP section in the config to return a Run_Step object
:param config_obj: ConfigParser objection
:param section: Section name
:return: an initialized Run_Step object


init_ts_start = get_standardized_timestamp('<STR_LIT>', None)<EOL>ts_start = init_ts_start<EOL>ts_end = '<STR_LIT:0>'<EOL>for run_step in run_steps:<EOL><INDENT>if run_step.ts_start and run_step.ts_end:<EOL><INDENT>if run_step.ts_start < ts_start:<EOL><INDENT>ts_start = run_step.ts_start<EOL><DEDENT>if run_step.ts_end > ts_end:<EOL><INDENT>ts_end = run_step.ts_end<EOL><DEDENT><DEDENT><DEDENT>if ts_end == '<STR_LIT:0>':<EOL><INDENT>ts_end = None<EOL><DEDENT>if ts_start == init_ts_start:<EOL><INDENT>ts_start = None<EOL><DEDENT>logger.info('<STR_LIT>' + str(ts_start) + '<STR_LIT>' + str(ts_end))<EOL>return ts_start, ts_end<EOL>

This method finds the time range which covers all the Run_Steps

:param run_steps: list of Run_Step objects
:return: tuple of start and end timestamps


def init_logging(logger, log_file, log_level):

with open(log_file, '<STR_LIT:w>'):<EOL><INDENT>pass<EOL><DEDENT>numeric_level = getattr(logging, log_level.upper(), None) if log_level else logging.INFO<EOL>if not isinstance(numeric_level, int):<EOL><INDENT>raise ValueError('<STR_LIT>' % log_level)<EOL><DEDENT>logger.setLevel(logging.DEBUG)<EOL>fh = logging.FileHandler(log_file)<EOL>fh.setLevel(logging.DEBUG)<EOL>ch = logging.StreamHandler()<EOL>ch.setLevel(numeric_level)<EOL>formatter = logging.Formatter('<STR_LIT>')<EOL>fh.setFormatter(formatter)<EOL>ch.setFormatter(formatter)<EOL>logger.addHandler(fh)<EOL>logger.addHandler(ch)<EOL>return CONSTANTS.OK<EOL>

Initialize the naarad logger.
:param: logger: logger object to initialize
:param: log_file: log file name
:param: log_level: log level (debug, info, warn, error)


infile = {}<EOL>aggr_hosts = None<EOL>aggr_metrics = None<EOL>ts_start = None<EOL>ts_end = None<EOL>precision = None<EOL>hostname = "<STR_LIT:localhost>"<EOL>rule_strings = {}<EOL>important_sub_metrics = None<EOL>anomaly_detection_metrics = None<EOL>try:<EOL><INDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>important_sub_metrics = config_obj.get(section, '<STR_LIT>').split()<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>hostname = config_obj.get(section, '<STR_LIT>')<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>infile = config_obj.get(section, '<STR_LIT>').split()<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>label = sanitize_string_section_name(section)<EOL>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>ts_start = get_standardized_timestamp(config_obj.get(section, '<STR_LIT>'), None)<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>ts_end = get_standardized_timestamp(config_obj.get(section, '<STR_LIT>'), None)<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>precision = config_obj.get(section, '<STR_LIT>')<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>aggr_hosts = config_obj.get(section, '<STR_LIT>')<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>aggr_metrics = config_obj.get(section, '<STR_LIT>')<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>anomaly_detection_metrics = config_obj.get(section, '<STR_LIT>').split()<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>rule_strings, other_options = get_rule_strings(config_obj, section)<EOL><DEDENT>except ConfigParser.NoOptionError:<EOL><INDENT>logger.exception("<STR_LIT>" + section)<EOL>sys.exit()<EOL><DEDENT>return (hostname, infile, aggr_hosts, aggr_metrics, label, ts_start, ts_end, precision, aggr_metrics, other_options,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics)<EOL>

Parse basic options from metric sections of the config
:param config_obj: ConfigParser object
:param section: Section name
:return: all the parsed options


regex = re.compile(r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>', re.IGNORECASE)<EOL>if regex.match(url):<EOL><INDENT>logger.info("<STR_LIT>")<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Check if a given string is in the correct URL format or not

:param str url:
:return: True or False


rule_strings = {}<EOL>config_obj = ConfigParser.ConfigParser()<EOL>config_obj.optionxform = str<EOL>config_obj.read(options_file)<EOL>for section in config_obj.sections():<EOL><INDENT>rule_strings, kwargs = get_rule_strings(config_obj, section)<EOL>for (key, val) in rule_strings.iteritems():<EOL><INDENT>set_sla(obj, section, key, val)<EOL><DEDENT><DEDENT>

Helper function to parse diff config file, which contains SLA rules for diff comparisons


if os.path.exists(filename):<EOL><INDENT>if not os.path.getsize(filename):<EOL><INDENT>logger.warning('<STR_LIT>', filename)<EOL>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logger.warning('<STR_LIT>', filename)<EOL>return False<EOL><DEDENT>return True<EOL>

Check if the specifed file exists and is not empty

:param filename: full path to the file that needs to be checked
:return: Status, Message


def initialize_aggregate_metric(section, aggr_hosts, aggr_metrics, metrics, outdir_default, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, other_options):

metric = None<EOL>metric_type = section.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>metric = aggregate_metric_classes[metric_type](section, aggr_hosts, aggr_metrics, metrics, outdir_default, resource_path, label, ts_start, ts_end,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)<EOL>return metric<EOL>

Initialize aggregate metric
:param: section: config section name
:param: aggr_hosts: list of hostnames to aggregate
:param: aggr_metrics: list of metrics to aggregate
:param: metrics: list of metric objects associated with the current naarad analysis
:param: outdir_default: report location
:param: resource_path: resource path for report
:param: label: label for config section
:param: ts_start: start time for analysis
:param: ts_end: end time for analysis
:param: rule_strings: list of slas
:param: important_sub_metrics: list of important sub metrics
:param: other_options: kwargs
:return: metric object


def initialize_metric(section, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, other_options):

metric = None<EOL>metric_type = section.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>if metric_type in metric_classes:<EOL><INDENT>if '<STR_LIT>' in metric_type:<EOL><INDENT>metric = metric_classes['<STR_LIT>'](section, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)<EOL><DEDENT>else:<EOL><INDENT>metric = metric_classes[metric_type](section, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>metric = Metric(section, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, **other_options)<EOL><DEDENT>return metric<EOL>

Initialize appropriate metric based on type of metric.
:param: section: config section name or auto discovered metric type
:param: infile_list: list of input log files for the metric
:param: hostname: hostname associated with the logs origin
:param: output_directory: report location
:param: resource_path: resource path for report
:param: label: label for config section or auto discovered metric type
:param: ts_start: start time for analysis
:param: ts_end: end time for analysis
:param: rule_strings: list of slas
:param: important_sub_metrics: list of important sub metrics
:param: anomaly_detection_metrics: list of metrics to use for anomaly detection.
:param: other_options: kwargs
:return: metric object


if not timestamp:<EOL><INDENT>return None<EOL><DEDENT>if timestamp == '<STR_LIT>':<EOL><INDENT>timestamp = str(datetime.datetime.now())<EOL><DEDENT>if not ts_format:<EOL><INDENT>ts_format = detect_timestamp_format(timestamp)<EOL><DEDENT>try:<EOL><INDENT>if ts_format == '<STR_LIT>':<EOL><INDENT>logger.error('<STR_LIT>', timestamp)<EOL>return -<NUM_LIT:1><EOL><DEDENT>elif ts_format == '<STR_LIT>':<EOL><INDENT>ts = int(timestamp) * <NUM_LIT:1000><EOL><DEDENT>elif ts_format == '<STR_LIT>':<EOL><INDENT>ts = timestamp<EOL><DEDENT>elif ts_format == '<STR_LIT>':<EOL><INDENT>ts = int(timestamp[:<NUM_LIT:10>]) * <NUM_LIT:1000> + int(timestamp[<NUM_LIT:11>:])<EOL><DEDENT>elif ts_format in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>date_today = str(datetime.date.today())<EOL>dt_obj = datetime.datetime.strptime(date_today + '<STR_LIT:U+0020>' + timestamp, '<STR_LIT>' + ts_format)<EOL>ts = calendar.timegm(dt_obj.utctimetuple()) * <NUM_LIT:1000> + dt_obj.microsecond / <NUM_LIT:1000><EOL><DEDENT>else:<EOL><INDENT>dt_obj = datetime.datetime.strptime(timestamp, ts_format)<EOL>ts = calendar.timegm(dt_obj.utctimetuple()) * <NUM_LIT:1000> + dt_obj.microsecond / <NUM_LIT:1000><EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>return str(ts)<EOL>

Given a timestamp string, return a time stamp in the epoch ms format. If no date is present in
timestamp then today's date will be added as a prefix before conversion to epoch ms


def parse_metric_section(config_obj, section, metric_classes, metrics, aggregate_metric_classes, outdir_default, resource_path):

(hostname, infile, aggr_hosts, aggr_metrics, label, ts_start, ts_end, precision, aggr_metrics, other_options,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics) = parse_basic_metric_options(config_obj, section)<EOL>metric_type = section.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>if metric_type in aggregate_metric_classes:<EOL><INDENT>new_metric = initialize_aggregate_metric(section, aggr_hosts, aggr_metrics, metrics, outdir_default, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, other_options)<EOL><DEDENT>else:<EOL><INDENT>new_metric = initialize_metric(section, infile, hostname, aggr_metrics, outdir_default, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, other_options)<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT:ignore>') and config_obj.getint(section, '<STR_LIT:ignore>') == <NUM_LIT:1>:<EOL><INDENT>new_metric.ignore = True<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>new_metric.calc_metrics = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>new_metric.precision = precision<EOL>return new_metric<EOL>

Parse a metric section and create a Metric object
:param config_obj: ConfigParser object
:param section: Section name
:param metric_classes: List of valid metric types
:param metrics: List of all regular metric objects (used by aggregate metric)
:param aggregate_metric_classes: List of all valid aggregate metric types
:param outdir_default: Default output directory
:param resource_path: Default resource directory
:return: An initialized Metric object


reg = re.compile('<STR_LIT>')<EOL>if reg.match(metric_name) and not metric_name.startswith('<STR_LIT:.>'):<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

check the validity of metric_name in config; the metric_name will be used for creation of sub-dir, so only contains: alphabet, digits , '.', '-' and '_'
:param str metric_name: metric_name
:return: True if valid


time_formats = {<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>')<EOL>}<EOL>for time_format in time_formats:<EOL><INDENT>if re.match(time_formats[time_format], timestamp):<EOL><INDENT>return time_format<EOL><DEDENT><DEDENT>return '<STR_LIT>'<EOL>

Given an input timestamp string, determine what format is it likely in.

:param string timestamp: the timestamp string for which we need to determine format
:return: best guess timestamp format


def parse_graph_section(config_obj, section, outdir_default, indir_default):

graph_timezone = None<EOL>graphing_library = CONSTANTS.DEFAULT_GRAPHING_LIBRARY<EOL>crossplots = []<EOL>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>graphing_library = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>graphs_string = config_obj.get(section, '<STR_LIT>')<EOL>crossplots = graphs_string.split()<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>outdir_default = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>outdir_default = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>indir_default = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>graph_timezone = config_obj.get(section, '<STR_LIT>')<EOL>if graph_timezone not in ("<STR_LIT>", "<STR_LIT>", "<STR_LIT>"):<EOL><INDENT>logger.warn('<STR_LIT>' + graph_timezone + '<STR_LIT>')<EOL>graph_timezone = "<STR_LIT>"<EOL><DEDENT><DEDENT>return graphing_library, crossplots, outdir_default, indir_default, graph_timezone<EOL>

Parse the GRAPH section of the config to extract useful values
:param config_obj: ConfigParser object
:param section: Section name
:param outdir_default: Default output directory passed in args
:param indir_default: Default input directory passed in args
:return: List of options extracted from the GRAPH section


try:<EOL><INDENT>(local_file, headers) = urllib.urlretrieve(url)<EOL><DEDENT>except:<EOL><INDENT>sys.exit("<STR_LIT>" + url + "<STR_LIT>")<EOL><DEDENT>return local_file<EOL>

Download a file pointed to by url to a temp file on local disk

:param str url:
:return: local_file


for url in url_list:<EOL><INDENT>download_url_single(url, outdir)<EOL><DEDENT>

Downloads list of http(s) urls to local files
:param list url_list: list of URLs to download
:param str outdir: Required. the local directory to put the downloadedfiles.
  If this is not given, then the local file will be the original one, as given in url.
:return None


def download_url_single(inputs, outdir, outfile=None):

if not inputs or type(inputs) != str or not outdir or type(outdir) != str:<EOL><INDENT>logging.error("<STR_LIT>")<EOL>return<EOL><DEDENT>else:<EOL><INDENT>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT><DEDENT>output_file = handle_single_url(inputs, outdir, outfile)<EOL>return output_file<EOL>

Downloads a http(s) url to a local file
:param str inputs:  the absolute url
:param str outdir: Required. the local directory to put the downloadedfiles.
:param str outfile: // Optional. If this is given, the downloaded url will be renated to outfile;
  If this is not given, then the local file will be the original one, as given in url.
:return: the local full path name of downloaded url


cmd_args = shlex.split(self.run_cmd)<EOL>logger.info('<STR_LIT>', self.run_rank)<EOL>logger.info('<STR_LIT>' + str(cmd_args))<EOL>self.ts_start = time.strftime("<STR_LIT>")<EOL>try:<EOL><INDENT>self.process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=<NUM_LIT:1>)<EOL>if self.kill_after_seconds:<EOL><INDENT>self.timer = Timer(self.kill_after_seconds, self.kill)<EOL>self.timer.start()<EOL><DEDENT>for line in iter(self.process.stdout.readline, b'<STR_LIT>'):<EOL><INDENT>logger.info(line.strip())<EOL><DEDENT>self.process.communicate()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>logger.warning('<STR_LIT>')<EOL>self.kill()<EOL><DEDENT>if self.timer:<EOL><INDENT>self.timer.cancel()<EOL><DEDENT>self.ts_end = time.strftime("<STR_LIT>")<EOL>logger.info('<STR_LIT>')<EOL>logger.info('<STR_LIT>' + self.ts_start + '<STR_LIT>' + self.ts_end)<EOL>

Run the command, infer time period to be used in metric analysis phase.
:return: None


def __init__(self, run_type, run_cmd, call_type, run_order, run_rank, should_wait=True, kill_after_seconds=None):

self.run_type = run_type<EOL>self.run_cmd = run_cmd<EOL>self.call_type = call_type<EOL>self.run_order = run_order<EOL>self.run_rank = run_rank<EOL>self.should_wait = should_wait<EOL>self.kill_after_seconds = kill_after_seconds<EOL>self.timer = None<EOL>

Init method
:param run_type: Type of run_step: "workload" only for now
:param run_cmd: Details of command to be run. It could be a command or API call
:param call_type: Kind of call -- local or remote
:param run_order: When to run this w.r.t analysis. One of ('pre', 'in', 'post')
:param run_rank: In what order to run this
:param should_wait: Boolean whether naarad should wait for the run command to finish or not before moving on
:param kill_after_seconds: Seconds for which the command should be run before being killed
:return: None


s = cls()<EOL>s.feed(html)<EOL>unescaped_data = s.unescape(s.get_data())<EOL>return escape_for_xml(unescaped_data, tags_to_keep=s.mathml_elements)<EOL>

Return stripped HTML, keeping only MathML.


self.fed.append('<STR_LIT>' % name)<EOL>

Return representation of numeric entities.


if tag in self.mathml_elements:<EOL><INDENT>self.fed.append("<STR_LIT>".format(tag))<EOL><DEDENT>


self.fed.append(d)<EOL>

Return representation of pure text data.


page = urllib2.urlopen(url)<EOL>pages = [BeautifulSoup(page)]<EOL>numpag = pages[<NUM_LIT:0>].body.findAll('<STR_LIT>', attrs={'<STR_LIT:class>': '<STR_LIT>'})<EOL>if len(numpag) > <NUM_LIT:0>:<EOL><INDENT>if re.search('<STR_LIT>', numpag[<NUM_LIT:0>].string):<EOL><INDENT>for i in range(int(numpag[<NUM_LIT:0>].string)-<NUM_LIT:1>):<EOL><INDENT>page = urllib2.urlopen('<STR_LIT>' % (url, i+<NUM_LIT:2>))<EOL>pages.append(BeautifulSoup(page))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>" % (numpag[<NUM_LIT:0>].string))<EOL><DEDENT><DEDENT>impl = getDOMImplementation()<EOL>doc = impl.createDocument(None, "<STR_LIT>", None)<EOL>links = []<EOL>for page in pages:<EOL><INDENT>links += page.body.findAll('<STR_LIT:p>', attrs={'<STR_LIT:class>': '<STR_LIT:title>'})<EOL>links += page.body.findAll('<STR_LIT>', attrs={'<STR_LIT:class>': '<STR_LIT:title>'})<EOL><DEDENT>for link in links:<EOL><INDENT>record = self._get_record(link)<EOL>doc.firstChild.appendChild(record)<EOL><DEDENT>return doc.toprettyxml()<EOL>

Returns the records listed in the webpage given as
parameter as a xml String.

@param url: the url of the Journal, Book, Protocol or Reference work


subfields = record_get_subfields(<EOL>rec, tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>try:<EOL><INDENT>del subfields[subfield_position]<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>{"<STR_LIT>": subfield_position,<EOL>"<STR_LIT>": str(field_position_local),<EOL>"<STR_LIT>": str(field_position_global),<EOL>"<STR_LIT>": tag})<EOL><DEDENT>if not subfields:<EOL><INDENT>if field_position_global is not None:<EOL><INDENT>for position, field in enumerate(rec[tag]):<EOL><INDENT>if field[<NUM_LIT:4>] == field_position_global:<EOL><INDENT>del rec[tag][position]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>del rec[tag][field_position_local]<EOL><DEDENT>if not rec[tag]:<EOL><INDENT>del rec[tag]<EOL><DEDENT><DEDENT>

Delete subfield from position specified.

Specify the subfield by tag, field number and subfield position.


def record_add_fields(rec, tag, fields, field_position_local=None,<EOL>field_position_global=None):

if field_position_local is None and field_position_global is None:<EOL><INDENT>for field in fields:<EOL><INDENT>record_add_field(<EOL>rec, tag, ind1=field[<NUM_LIT:1>],<EOL>ind2=field[<NUM_LIT:2>], subfields=field[<NUM_LIT:0>],<EOL>controlfield_value=field[<NUM_LIT:3>])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fields.reverse()<EOL>for field in fields:<EOL><INDENT>record_add_field(<EOL>rec, tag, ind1=field[<NUM_LIT:1>], ind2=field[<NUM_LIT:2>],<EOL>subfields=field[<NUM_LIT:0>], controlfield_value=field[<NUM_LIT:3>],<EOL>field_position_local=field_position_local,<EOL>field_position_global=field_position_global)<EOL><DEDENT><DEDENT>return field_position_local<EOL>

Add the fields into the record at the required position.

The position is specified by the tag and the field_position_local in the
list of fields.

:param rec: a record structure
:param tag: the tag of the fields to be moved
:param field_position_local: the field_position_local to which the field
                             will be inserted. If not specified, appends
                             the fields to the tag.
:param a: list of fields to be added
:return: -1 if the operation failed, or the field_position_local if it was
         successful


def field_add_subfield(field, code, value):

field[<NUM_LIT:0>].append((code, value))<EOL>


field_dict = {}<EOL>field_positions_global = []<EOL>for field in fields:<EOL><INDENT>field_dict.setdefault(field[<NUM_LIT:1>:<NUM_LIT:3>], []).append(field)<EOL>field_positions_global.append(field[<NUM_LIT:4>])<EOL><DEDENT>indicators = list(field_dict.keys())<EOL>indicators.sort()<EOL>field_list = []<EOL>for indicator in indicators:<EOL><INDENT>for field in field_dict[indicator]:<EOL><INDENT>field_list.append(field[:<NUM_LIT:4>] + (field_positions_global.pop(<NUM_LIT:0>),))<EOL><DEDENT><DEDENT>return field_list<EOL>

Sort a set of fields by their indicators.

    Return a sorted list with correct global field positions.


def record_get_field_instances(rec, tag="<STR_LIT>", ind1="<STR_LIT:U+0020>", ind2="<STR_LIT:U+0020>"):

if not rec:<EOL><INDENT>return []<EOL><DEDENT>if not tag:<EOL><INDENT>return list(rec.items())<EOL><DEDENT>else:<EOL><INDENT>out = []<EOL>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>if '<STR_LIT:%>' in tag:<EOL><INDENT>for field_tag in rec:<EOL><INDENT>if _tag_matches_pattern(field_tag, tag):<EOL><INDENT>for possible_field_instance in rec[field_tag]:<EOL><INDENT>if (ind1 in ('<STR_LIT:%>', possible_field_instance[<NUM_LIT:1>]) and<EOL>ind2 in ('<STR_LIT:%>', possible_field_instance[<NUM_LIT:2>])):<EOL><INDENT>out.append(possible_field_instance)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for possible_field_instance in rec.get(tag, []):<EOL><INDENT>if (ind1 in ('<STR_LIT:%>', possible_field_instance[<NUM_LIT:1>]) and<EOL>ind2 in ('<STR_LIT:%>', possible_field_instance[<NUM_LIT:2>])):<EOL><INDENT>out.append(possible_field_instance)<EOL><DEDENT><DEDENT><DEDENT>return out<EOL><DEDENT>

Return the list of field instances for the specified tag and indications.

Return empty list if not found.
If tag is empty string, returns all fields

Parameters (tag, ind1, ind2) can contain wildcard %.

:param rec: a record structure as returned by create_record()
:param tag: a 3 characters long string
:param ind1: a 1 character long string
:param ind2: a 1 character long string
:param code: a 1 character long string
:return: a list of field tuples (Subfields, ind1, ind2, value,
         field_position_global) where subfields is list of (code, value)


errors = []<EOL>for tag in list(record.keys()):<EOL><INDENT>upper_bound = '<STR_LIT>'<EOL>n = len(tag)<EOL>if n > <NUM_LIT:3>:<EOL><INDENT>i = n - <NUM_LIT:3><EOL>while i > <NUM_LIT:0>:<EOL><INDENT>upper_bound = '<STR_LIT>' % ('<STR_LIT:0>', upper_bound)<EOL>i -= <NUM_LIT:1><EOL><DEDENT><DEDENT>if tag == '<STR_LIT:!>':<EOL><INDENT>errors.append((<NUM_LIT:1>, '<STR_LIT>' +<EOL>str([f[<NUM_LIT:4>] for f in record[tag]]) + '<STR_LIT:)>'))<EOL>record['<STR_LIT>'] = record.pop(tag)<EOL>tag = '<STR_LIT>'<EOL><DEDENT>elif not ('<STR_LIT>' <= tag <= upper_bound or<EOL>tag in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>')):<EOL><INDENT>errors.append(<NUM_LIT:2>)<EOL>record['<STR_LIT>'] = record.pop(tag)<EOL>tag = '<STR_LIT>'<EOL><DEDENT>fields = []<EOL>for field in record[tag]:<EOL><INDENT>if field[<NUM_LIT:0>] == [] and field[<NUM_LIT:3>] == '<STR_LIT>':<EOL><INDENT>errors.append((<NUM_LIT:8>, '<STR_LIT>' + str(field[<NUM_LIT:4>]) + '<STR_LIT:)>'))<EOL><DEDENT>subfields = []<EOL>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == '<STR_LIT:!>':<EOL><INDENT>errors.append((<NUM_LIT:3>, '<STR_LIT>' + str(field[<NUM_LIT:4>]) + '<STR_LIT:)>'))<EOL>newsub = ('<STR_LIT>', subfield[<NUM_LIT:1>])<EOL><DEDENT>else:<EOL><INDENT>newsub = subfield<EOL><DEDENT>subfields.append(newsub)<EOL><DEDENT>if field[<NUM_LIT:1>] == '<STR_LIT:!>':<EOL><INDENT>errors.append((<NUM_LIT:4>, '<STR_LIT>' + str(field[<NUM_LIT:4>]) + '<STR_LIT:)>'))<EOL>ind1 = "<STR_LIT:U+0020>"<EOL><DEDENT>else:<EOL><INDENT>ind1 = field[<NUM_LIT:1>]<EOL><DEDENT>if field[<NUM_LIT:2>] == '<STR_LIT:!>':<EOL><INDENT>errors.append((<NUM_LIT:5>, '<STR_LIT>' + str(field[<NUM_LIT:4>]) + '<STR_LIT:)>'))<EOL>ind2 = "<STR_LIT:U+0020>"<EOL><DEDENT>else:<EOL><INDENT>ind2 = field[<NUM_LIT:2>]<EOL><DEDENT>fields.append((subfields, ind1, ind2, field[<NUM_LIT:3>], field[<NUM_LIT:4>]))<EOL><DEDENT>record[tag] = fields<EOL><DEDENT>return errors<EOL>

Check and correct the structure of the record.

:param record: the record data structure
:return: a list of errors found


all_fields = []<EOL>for tag, fields in list(record.items()):<EOL><INDENT>previous_field_position_global = -<NUM_LIT:1><EOL>for field in fields:<EOL><INDENT>if field[<NUM_LIT:4>] < previous_field_position_global:<EOL><INDENT>return ("<STR_LIT>" %<EOL>tag)<EOL><DEDENT>previous_field_position_global = field[<NUM_LIT:4>]<EOL>if field[<NUM_LIT:4>] in all_fields:<EOL><INDENT>return ("<STR_LIT>" %<EOL>(field[<NUM_LIT:4>], tag))<EOL><DEDENT><DEDENT><DEDENT>

Check if the global field positions in the record are valid.

I.e., no duplicate global field positions and local field positions in the
list of fields are ascending.

:param record: the record data structure
:return: the first error found as a string or None if no error was found


def record_modify_subfield(rec, tag, subfield_code, value, subfield_position,<EOL>field_position_global=None,<EOL>field_position_local=None):

subfields = record_get_subfields(<EOL>rec, tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>try:<EOL><INDENT>subfields[subfield_position] = (subfield_code, value)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>" % subfield_position)<EOL><DEDENT>

Modify subfield at specified position.

    Specify the subfield by tag, field number and subfield position.


if not path_to_xml:<EOL><INDENT>if not self.path:<EOL><INDENT>self.logger.error("<STR_LIT>")<EOL>return<EOL><DEDENT>path_to_xml = self.path<EOL><DEDENT>root = self._clean_xml(path_to_xml)<EOL>if root.tag.lower() == '<STR_LIT>':<EOL><INDENT>tree = ET.ElementTree(root)<EOL>self.records = element_tree_collection_to_records(tree)<EOL><DEDENT>elif root.tag.lower() == '<STR_LIT>':<EOL><INDENT>new_root = ET.Element('<STR_LIT>')<EOL>new_root.append(root)<EOL>tree = ET.ElementTree(new_root)<EOL>self.records = element_tree_collection_to_records(tree)<EOL><DEDENT>else:<EOL><INDENT>header_subs = get_request_subfields(root)<EOL>records = root.find('<STR_LIT>')<EOL>if records is None:<EOL><INDENT>records = root.find('<STR_LIT>')<EOL><DEDENT>if records is None:<EOL><INDENT>raise ValueError("<STR_LIT>")<EOL><DEDENT>tree = ET.ElementTree(records)<EOL>for record, is_deleted in element_tree_oai_records(tree, header_subs):<EOL><INDENT>if is_deleted:<EOL><INDENT>self.deleted_records.append(<EOL>self.create_deleted_record(record)<EOL>)<EOL><DEDENT>else:<EOL><INDENT>self.records.append(record)<EOL><DEDENT><DEDENT><DEDENT>

Parse an XML document and clean any namespaces.


def _shift_field_positions_global(record, start, delta=<NUM_LIT:1>):

if not delta:<EOL><INDENT>return<EOL><DEDENT>for tag, fields in list(record.items()):<EOL><INDENT>newfields = []<EOL>for field in fields:<EOL><INDENT>if field[<NUM_LIT:4>] < start:<EOL><INDENT>newfields.append(field)<EOL><DEDENT>else:<EOL><INDENT>newfields.append(tuple(list(field[:<NUM_LIT:4>]) + [field[<NUM_LIT:4>] + delta]))<EOL><DEDENT><DEDENT>record[tag] = newfields<EOL><DEDENT>

Shift all global field positions.

Shift all global field positions with global field positions
higher or equal to 'start' from the value 'delta'.


def record_match_subfields(rec, tag, ind1="<STR_LIT:U+0020>", ind2="<STR_LIT:U+0020>", sub_key=None,<EOL>sub_value='<STR_LIT>', sub_key2=None, sub_value2='<STR_LIT>',<EOL>case_sensitive=True):

if sub_key is None:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>if sub_key2 is not None and sub_value2 is '<STR_LIT>':<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>+ "<STR_LIT>")<EOL><DEDENT>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>if not case_sensitive:<EOL><INDENT>sub_value = sub_value.lower()<EOL>sub_value2 = sub_value2.lower()<EOL><DEDENT>for field in record_get_field_instances(rec, tag, ind1, ind2):<EOL><INDENT>subfields = dict(field_get_subfield_instances(field))<EOL>if not case_sensitive:<EOL><INDENT>for k, v in subfields.items():<EOL><INDENT>subfields[k] = v.lower()<EOL><DEDENT><DEDENT>if sub_key in subfields:<EOL><INDENT>if sub_value is '<STR_LIT>':<EOL><INDENT>return field[<NUM_LIT:4>]<EOL><DEDENT>else:<EOL><INDENT>if sub_value == subfields[sub_key]:<EOL><INDENT>if sub_key2 is None:<EOL><INDENT>return field[<NUM_LIT:4>]<EOL><DEDENT>else:<EOL><INDENT>if sub_key2 in subfields:<EOL><INDENT>if sub_value2 == subfields[sub_key2]:<EOL><INDENT>return field[<NUM_LIT:4>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return False<EOL>

Find subfield instances in a particular field.

It tests values in 1 of 3 possible ways:
 - Does a subfield code exist? (ie does 773__a exist?)
 - Does a subfield have a particular value? (ie 773__a == 'PhysX')
 - Do a pair of subfields have particular values?
    (ie 035__2 == 'CDS' and 035__a == '123456')

Parameters:
 * rec - dictionary: a bibrecord structure
 * tag - string: the tag of the field (ie '773')
 * ind1, ind2 - char: a single characters for the MARC indicators
 * sub_key - char: subfield key to find
 * sub_value - string: subfield value of that key
 * sub_key2 - char: key of subfield to compare against
 * sub_value2 - string: expected value of second subfield
 * case_sensitive - bool: be case sensitive when matching values

:return: false if no match found, else provides the field position (int)


def filter_field_instances(field_instances, filter_subcode, filter_value,<EOL>filter_mode='<STR_LIT:e>'):

matched = []<EOL>if filter_mode == '<STR_LIT:e>':<EOL><INDENT>to_match = (filter_subcode, filter_value)<EOL>for instance in field_instances:<EOL><INDENT>if to_match in instance[<NUM_LIT:0>]:<EOL><INDENT>matched.append(instance)<EOL><DEDENT><DEDENT><DEDENT>elif filter_mode == '<STR_LIT:s>':<EOL><INDENT>for instance in field_instances:<EOL><INDENT>for subfield in instance[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == filter_subcode andsubfield[<NUM_LIT:1>].find(filter_value) > -<NUM_LIT:1>:<EOL><INDENT>matched.append(instance)<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif filter_mode == '<STR_LIT:r>':<EOL><INDENT>reg_exp = re.compile(filter_value)<EOL>for instance in field_instances:<EOL><INDENT>for subfield in instance[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == filter_subcode andreg_exp.match(subfield[<NUM_LIT:1>]) is not None:<EOL><INDENT>matched.append(instance)<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return matched<EOL>

Filter the given field.

    Filters given field and returns only that field instances that contain
    filter_subcode with given filter_value. As an input for search function
    accepts output from record_get_field_instances function. Function can be
    run in three modes:

    - 'e' - looking for exact match in subfield value
    - 's' - looking for substring in subfield value
    - 'r' - looking for regular expression in subfield value

    Example:

    record_filter_field(record_get_field_instances(rec, '999', '%', '%'),
                        'y', '2001')

    In this case filter_subcode is 'y' and filter_value is '2001'.

    :param field_instances: output from record_get_field_instances
    :param filter_subcode: name of the subfield
    :type filter_subcode: string
    :param filter_value: value of the subfield
    :type filter_value: string
    :param filter_mode: 'e','s' or 'r'


return tag in rec<EOL>

Check if the tag exists in the record.

:param rec: the record data structure
:param the: field
:return: a boolean


return [element for single_list in alist for element in single_list]<EOL>


def record_move_subfield(rec, tag, subfield_position, new_subfield_position,<EOL>field_position_global=None,<EOL>field_position_local=None):

subfields = record_get_subfields(<EOL>rec,<EOL>tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>try:<EOL><INDENT>subfield = subfields.pop(subfield_position)<EOL>subfields.insert(new_subfield_position, subfield)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>" % subfield_position)<EOL><DEDENT>

Move subfield at specified position.

    Sspecify the subfield by tag, field number and subfield position to new
    subfield position.


def record_move_fields(rec, tag, field_positions_local,<EOL>field_position_local=None):

fields = record_delete_fields(<EOL>rec, tag,<EOL>field_positions_local=field_positions_local)<EOL>return record_add_fields(<EOL>rec, tag, fields,<EOL>field_position_local=field_position_local)<EOL>

Move some fields to the position specified by 'field_position_local'.

:param rec: a record structure as returned by create_record()
:param tag: the tag of the fields to be moved
:param field_positions_local: the positions of the fields to move
:param field_position_local: insert the field before that
                             field_position_local. If unspecified, appends
                             the fields :return: the field_position_local
                             is the operation was successful


for tag, fields in list(record.items()):<EOL><INDENT>record[tag] = _fields_sort_by_indicators(fields)<EOL><DEDENT>

Sort the fields inside the record by indicators.


for tag in list(rec.keys()):<EOL><INDENT>if tag[:<NUM_LIT:2>] == '<STR_LIT>' and rec[tag][<NUM_LIT:0>][<NUM_LIT:3>]:<EOL><INDENT>del rec[tag]<EOL><DEDENT><DEDENT>

Remove all non-empty controlfields from the record.

:param rec:  A record dictionary structure
:type  rec:  dictionary


marcxml = []<EOL>if field[<NUM_LIT:3>]:<EOL><INDENT>marcxml.append('<STR_LIT>' %<EOL>(tag, MathMLParser.html_to_text(field[<NUM_LIT:3>])))<EOL><DEDENT>else:<EOL><INDENT>marcxml.append('<STR_LIT>' %<EOL>(tag, field[<NUM_LIT:1>], field[<NUM_LIT:2>]))<EOL>marcxml += [_subfield_xml_output(subfield) for subfield in field[<NUM_LIT:0>]]<EOL>marcxml.append('<STR_LIT>')<EOL><DEDENT>return '<STR_LIT:\n>'.join(marcxml)<EOL>

Generate the XML for field 'field' and returns it as a string.


if isinstance(code, str):<EOL><INDENT>return code<EOL><DEDENT>message = '<STR_LIT>'<EOL>if isinstance(code, tuple):<EOL><INDENT>if isinstance(code[<NUM_LIT:0>], str):<EOL><INDENT>message = code[<NUM_LIT:1>]<EOL>code = code[<NUM_LIT:0>]<EOL><DEDENT><DEDENT>return CFG_BIBRECORD_WARNING_MSGS.get(code, '<STR_LIT>') + message<EOL>

Return a warning message of code 'code'.

If code = (cd, str) it returns the warning message of code 'cd' and appends
str at the end


return self.records<EOL>


def create_field(subfields=None, ind1='<STR_LIT:U+0020>', ind2='<STR_LIT:U+0020>', controlfield_value='<STR_LIT>',<EOL>global_position=-<NUM_LIT:1>):

if subfields is None:<EOL><INDENT>subfields = []<EOL><DEDENT>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>field = (subfields, ind1, ind2, controlfield_value, global_position)<EOL>_check_field_validity(field)<EOL>return field<EOL>

Return a field created with the provided elements.

Global position is set arbitrary to -1.


return cmp(field1[<NUM_LIT:0>], field2[<NUM_LIT:0>])<EOL>

Function used to order the fields according to the tags.


pairs = {}<EOL>for key, value in field[<NUM_LIT:0>]:<EOL><INDENT>if key in pairs and pairs[key] != value:<EOL><INDENT>pairs[key].append(value)<EOL><DEDENT>else:<EOL><INDENT>pairs[key] = [value]<EOL><DEDENT><DEDENT>return pairs<EOL>

Given a field, will place all subfields into a dictionary
    Parameters:
     * field - tuple: The field to get subfields for
    Returns: a dictionary, codes as keys and a list of values as the value


out = {}<EOL>position = <NUM_LIT:0><EOL>tags = sorted(record.keys())<EOL>for tag in tags:<EOL><INDENT>fields = record[tag]<EOL>out[tag] = []<EOL>current_fields = set()<EOL>for full_field in fields:<EOL><INDENT>field = (tuple(full_field[<NUM_LIT:0>]),) + full_field[<NUM_LIT:1>:<NUM_LIT:4>]<EOL>if field not in current_fields:<EOL><INDENT>current_fields.add(field)<EOL>position += <NUM_LIT:1><EOL>out[tag].append(full_field[:<NUM_LIT:4>] + (position,))<EOL><DEDENT><DEDENT><DEDENT>return out<EOL>

Return a record where all the duplicate fields have been removed.

Fields are considered identical considering also the order of their
subfields.


parser = etree.XMLParser(dtd_validation=correct,<EOL>recover=(verbose <= <NUM_LIT:3>))<EOL>if correct:<EOL><INDENT>marcxml = '<STR_LIT>''<STR_LIT>' % (marcxml,)<EOL><DEDENT>try:<EOL><INDENT>tree = etree.parse(StringIO(marcxml), parser)<EOL><DEDENT>except Exception as e:<EOL><INDENT>raise InvenioBibRecordParserError(str(e))<EOL><DEDENT>record = {}<EOL>field_position_global = <NUM_LIT:0><EOL>controlfield_iterator = tree.iter(tag='<STR_LIT>')<EOL>for controlfield in controlfield_iterator:<EOL><INDENT>tag = controlfield.attrib.get('<STR_LIT>', '<STR_LIT:!>').encode("<STR_LIT>")<EOL>ind1 = '<STR_LIT:U+0020>'<EOL>ind2 = '<STR_LIT:U+0020>'<EOL>text = controlfield.text<EOL>if text is None:<EOL><INDENT>text = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>text = text.encode("<STR_LIT>")<EOL><DEDENT>subfields = []<EOL>if text or keep_singletons:<EOL><INDENT>field_position_global += <NUM_LIT:1><EOL>record.setdefault(tag, []).append((subfields, ind1, ind2, text,<EOL>field_position_global))<EOL><DEDENT><DEDENT>datafield_iterator = tree.iter(tag='<STR_LIT>')<EOL>for datafield in datafield_iterator:<EOL><INDENT>tag = datafield.attrib.get('<STR_LIT>', '<STR_LIT:!>').encode("<STR_LIT>")<EOL>ind1 = datafield.attrib.get('<STR_LIT>', '<STR_LIT:!>').encode("<STR_LIT>")<EOL>ind2 = datafield.attrib.get('<STR_LIT>', '<STR_LIT:!>').encode("<STR_LIT>")<EOL>if ind1 in ('<STR_LIT>', '<STR_LIT:_>'):<EOL><INDENT>ind1 = '<STR_LIT:U+0020>'<EOL><DEDENT>if ind2 in ('<STR_LIT>', '<STR_LIT:_>'):<EOL><INDENT>ind2 = '<STR_LIT:U+0020>'<EOL><DEDENT>subfields = []<EOL>subfield_iterator = datafield.iter(tag='<STR_LIT>')<EOL>for subfield in subfield_iterator:<EOL><INDENT>code = subfield.attrib.get('<STR_LIT:code>', '<STR_LIT:!>').encode("<STR_LIT>")<EOL>text = subfield.text<EOL>if text is None:<EOL><INDENT>text = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>text = text.encode("<STR_LIT>")<EOL><DEDENT>if text or keep_singletons:<EOL><INDENT>subfields.append((code, text))<EOL><DEDENT><DEDENT>if subfields or keep_singletons:<EOL><INDENT>text = '<STR_LIT>'<EOL>field_position_global += <NUM_LIT:1><EOL>record.setdefault(tag, []).append((subfields, ind1, ind2, text,<EOL>field_position_global))<EOL><DEDENT><DEDENT>return record<EOL>

Create a record object using the LXML parser.

If correct == 1, then perform DTD validation
If correct == 0, then do not perform DTD validation

If verbose == 0, the parser will not give warnings.
If 1 <= verbose <= 3, the parser will not give errors, but will warn
    the user about possible mistakes (implement me!)
If verbose > 3 then the parser will be strict and will stop in case of
    well-formedness errors or DTD errors.


for key in rec.keys():<EOL><INDENT>if key not in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>


out = []<EOL>if node:<EOL><INDENT>for child in node:<EOL><INDENT>if child.nodeType == child.TEXT_NODE:<EOL><INDENT>out.append(child.data)<EOL><DEDENT>else:<EOL><INDENT>out.append(_get_children_as_string(child.childNodes))<EOL><DEDENT><DEDENT><DEDENT>return '<STR_LIT>'.join(out)<EOL>

Iterate through all the children of a node.

    Returns one string containing the values from all the text-nodes
    recursively.


def record_delete_fields(rec, tag, field_positions_local=None):

if tag not in rec:<EOL><INDENT>return []<EOL><DEDENT>new_fields, deleted_fields = [], []<EOL>for position, field in enumerate(rec.get(tag, [])):<EOL><INDENT>if field_positions_local is None or position in field_positions_local:<EOL><INDENT>deleted_fields.append(field)<EOL><DEDENT>else:<EOL><INDENT>new_fields.append(field)<EOL><DEDENT><DEDENT>if new_fields:<EOL><INDENT>rec[tag] = new_fields<EOL><DEDENT>else:<EOL><INDENT>del rec[tag]<EOL><DEDENT>return deleted_fields<EOL>

Delete all/some fields defined with MARC tag 'tag' from record 'rec'.

:param rec: a record structure.
:type rec: tuple
:param tag: three letter field.
:type tag: string
:param field_position_local: if set, it is the list of local positions
    within all the fields with the specified tag, that should be deleted.
    If not set all the fields with the specified tag will be deleted.
:type field_position_local: sequence
:return: the list of deleted fields.
:rtype: list
:note: the record is modified in place.


newl = []<EOL>for l in alist:<EOL><INDENT>newl.extend(l)<EOL><DEDENT>return newl<EOL>


return [subfield_value<EOL>for subfield_code, subfield_value in field_instance[<NUM_LIT:0>]<EOL>if subfield_code == code]<EOL>

Return subfield CODE values of the field instance FIELD.


def print_rec(rec, format=<NUM_LIT:1>, tags=None):

if tags is None:<EOL><INDENT>tags = []<EOL><DEDENT>if format == <NUM_LIT:1>:<EOL><INDENT>text = record_xml_output(rec, tags)<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return text<EOL>

Print a record.

:param format: 1 XML, 2 HTML (not implemented)
:param tags: list of tags to be printed


def _compare_fields(field1, field2, strict=True):

if strict:<EOL><INDENT>return field1[:<NUM_LIT:4>] == field2[:<NUM_LIT:4>]<EOL><DEDENT>else:<EOL><INDENT>if field1[<NUM_LIT:1>:<NUM_LIT:4>] != field2[<NUM_LIT:1>:<NUM_LIT:4>]:<EOL><INDENT>return False<EOL><DEDENT>else:<EOL><INDENT>return set(field1[<NUM_LIT:0>]) == set(field2[<NUM_LIT:0>])<EOL><DEDENT><DEDENT>

Compare 2 fields.

If strict is True, then the order of the subfield will be taken care of, if
not then the order of the subfields doesn't matter.

:return: True if the field are equivalent, False otherwise.


def record_get_field(rec, tag, field_position_global=None,<EOL>field_position_local=None):

if field_position_global is None and field_position_local is None:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>elif field_position_global is not None andfield_position_local is not None:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>elif field_position_global:<EOL><INDENT>if tag not in rec:<EOL><INDENT>raise InvenioBibRecordFieldError("<STR_LIT>" % tag)<EOL><DEDENT>for field in rec[tag]:<EOL><INDENT>if field[<NUM_LIT:4>] == field_position_global:<EOL><INDENT>return field<EOL><DEDENT><DEDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (tag, field_position_global))<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>return rec[tag][field_position_local]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise InvenioBibRecordFieldError("<STR_LIT>" % tag)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (tag, field_position_local))<EOL><DEDENT><DEDENT>

Return the the matching field.

One has to enter either a global field position or a local field position.

:return: a list of subfield tuples (subfield code, value).
:rtype: list


return [_warning(element) for element in alist]<EOL>

Apply the function _warning() to every element in alist.


def record_delete_subfield(rec, tag, subfield_code, ind1='<STR_LIT:U+0020>', ind2='<STR_LIT:U+0020>'):

ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>for field in rec.get(tag, []):<EOL><INDENT>if field[<NUM_LIT:1>] == ind1 and field[<NUM_LIT:2>] == ind2:<EOL><INDENT>field[<NUM_LIT:0>][:] = [subfield for subfield in field[<NUM_LIT:0>]<EOL>if subfield_code != subfield[<NUM_LIT:0>]]<EOL><DEDENT><DEDENT>

Delete all subfields with subfield_code in the record.


field = record_get_field(<EOL>rec, tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>new_field = (field[<NUM_LIT:0>], field[<NUM_LIT:1>], field[<NUM_LIT:2>], controlfield_value, field[<NUM_LIT:4>])<EOL>record_replace_field(<EOL>rec, tag, new_field,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>

Modify controlfield at position specified by tag and field number.


for char1, char2 in zip(tag, pattern):<EOL><INDENT>if char2 not in ('<STR_LIT:%>', char1):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>return True<EOL>

Return true if MARC 'tag' matches a 'pattern'.

    'pattern' is plain text, with % as wildcard

    Both parameters must be 3 characters long strings.

    .. doctest::

        >>> _tag_matches_pattern("909", "909")
        True
        >>> _tag_matches_pattern("909", "9%9")
        True
        >>> _tag_matches_pattern("909", "9%8")
        False

    :param tag: a 3 characters long string
    :param pattern: a 3 characters long string
    :return: False or True


identifier = record_get_field_value(record,<EOL>tag="<STR_LIT>",<EOL>code="<STR_LIT:a>")<EOL>recid = identifier.split("<STR_LIT::>")[-<NUM_LIT:1>]<EOL>try:<EOL><INDENT>source = identifier.split("<STR_LIT::>")[<NUM_LIT:1>]<EOL><DEDENT>except IndexError:<EOL><INDENT>source = "<STR_LIT>"<EOL><DEDENT>record_add_field(record, "<STR_LIT>",<EOL>subfields=[("<STR_LIT>", source), ("<STR_LIT:a>", recid)])<EOL>record_add_field(record, "<STR_LIT>",<EOL>subfields=[("<STR_LIT:c>", "<STR_LIT>")])<EOL>return record<EOL>

Generate the record deletion if deleted form OAI-PMH.


def get_record_rich(self, filename, ref_extract_callback=None):

self.document = parse(filename)<EOL>rec = create_record()<EOL>articles = self.document.getElementsByTagName('<STR_LIT>')<EOL>for article in articles:<EOL><INDENT>article_type = article.getAttribute('<STR_LIT>')<EOL>if not article_type == '<STR_LIT>':<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>doi = get_value_in_tag(self.document, '<STR_LIT>')<EOL>date = '<STR_LIT>'<EOL>for tag in self.document.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>year = get_value_in_tag(tag, '<STR_LIT>')<EOL>month = get_value_in_tag(tag, '<STR_LIT>').zfill(<NUM_LIT:2>)<EOL>day = get_value_in_tag(tag, '<STR_LIT>').zfill(<NUM_LIT:2>)<EOL>date = "<STR_LIT>" % (year, month, day)<EOL><DEDENT>if not date:<EOL><INDENT>for tag in self.document.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>year = get_value_in_tag(tag, '<STR_LIT>')<EOL>month = get_value_in_tag(tag, '<STR_LIT>').zfill(<NUM_LIT:2>)<EOL>day = get_value_in_tag(tag, '<STR_LIT>').zfill(<NUM_LIT:2>)<EOL>date = "<STR_LIT>" % (year, month, day)<EOL><DEDENT><DEDENT>first_page = get_value_in_tag(article, '<STR_LIT>')<EOL>last_page = get_value_in_tag(article, '<STR_LIT>')<EOL>subjects = article.getElementsByTagName('<STR_LIT>')<EOL>subjects = map(xml_to_text, subjects)<EOL>subject = '<STR_LIT:U+002CU+0020>'.join(subjects)<EOL>copyright_statement = get_value_in_tag(article, '<STR_LIT>')<EOL><DEDENT>journal = get_value_in_tag(self.document, '<STR_LIT>')<EOL>journal, volume = fix_journal_name(journal, self.journal_mappings)<EOL>issues = self.document.getElementsByTagName('<STR_LIT>')<EOL>for issue in issues:<EOL><INDENT>volume += get_value_in_tag(issue, '<STR_LIT>')<EOL>year = get_value_in_tag(issue, '<STR_LIT>')<EOL><DEDENT>title = get_value_in_tag(self.document, '<STR_LIT>')<EOL>authors = self.document.getElementsByTagName('<STR_LIT>')<EOL>affiliations = self.document.getElementsByTagName('<STR_LIT>')<EOL>def affiliation_pair(a):<EOL><INDENT>return a.getAttribute('<STR_LIT>'), get_value_in_tag(<EOL>a, '<STR_LIT>'<EOL>)<EOL><DEDENT>affiliations = map(affiliation_pair, affiliations)<EOL>affiliations = dict(affiliations)<EOL>def author_pair(a):<EOL><INDENT>surname = get_value_in_tag(a, '<STR_LIT>')<EOL>first_name = get_value_in_tag(a, '<STR_LIT>')<EOL>middle_name = get_value_in_tag(a, '<STR_LIT>')<EOL>if middle_name:<EOL><INDENT>name = '<STR_LIT>' % (surname, first_name, middle_name)<EOL><DEDENT>else:<EOL><INDENT>name = '<STR_LIT>' % (surname, first_name)<EOL><DEDENT>try:<EOL><INDENT>affid = a.getElementsByTagName(<EOL>'<STR_LIT>'<EOL>)[<NUM_LIT:0>].getAttribute('<STR_LIT>')<EOL>affiliation = affiliations[affid]<EOL><DEDENT>except IndexError:<EOL><INDENT>affiliation = '<STR_LIT>'<EOL><DEDENT>except KeyError:<EOL><INDENT>affiliation = '<STR_LIT>'<EOL><DEDENT>return name, affiliation<EOL><DEDENT>authors = map(author_pair, authors)<EOL>abstract = get_value_in_tag(self.document, '<STR_LIT>')<EOL>references = self.document.getElementsByTagName('<STR_LIT>')<EOL>for reference in references:<EOL><INDENT>subfields = []<EOL>label = reference.getAttribute('<STR_LIT:N>')<EOL>if label:<EOL><INDENT>subfields.append(('<STR_LIT:o>', label))<EOL><DEDENT>bibliosets = reference.getElementsByTagName('<STR_LIT>')<EOL>for tag in bibliosets:<EOL><INDENT>ref_year = get_value_in_tag(tag, '<STR_LIT>')<EOL>ref_journal = get_value_in_tag(tag, '<STR_LIT>')<EOL>ref_journal, ref_volume = fix_journal_name(<EOL>ref_journal, self.journal_mappings<EOL>)<EOL>ref_volume += get_value_in_tag(tag, '<STR_LIT>')<EOL>ref_page = get_value_in_tag(tag, '<STR_LIT>')<EOL>if ref_year:<EOL><INDENT>subfields.append(('<STR_LIT:y>', ref_year))<EOL><DEDENT>if ref_journal and ref_volume and ref_page:<EOL><INDENT>subfields.append(('<STR_LIT:s>', '<STR_LIT>' % (ref_journal,<EOL>ref_volume,<EOL>ref_page)))<EOL><DEDENT>reference.removeChild(tag)<EOL><DEDENT>text_ref = xml_to_text(reference)<EOL>if ref_extract_callback:<EOL><INDENT>ref_xml = ref_extract_callback(text_ref)<EOL>dom = parseString(ref_xml)<EOL>fields = dom.getElementsByTagName("<STR_LIT>")[<NUM_LIT:0>]<EOL>fields = fields.getElementsByTagName("<STR_LIT>")<EOL>if fields:<EOL><INDENT>subfields.append(('<STR_LIT>', '<STR_LIT>'))<EOL><DEDENT>for field in fields:<EOL><INDENT>data = field.firstChild.data<EOL>code = field.getAttribute("<STR_LIT:code>")<EOL>if code == '<STR_LIT:m>' and bibliosets:<EOL><INDENT>continue<EOL><DEDENT>else:<EOL><INDENT>subfields.append((code, data))<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>subfields.append(('<STR_LIT:m>', text_ref))<EOL><DEDENT>if subfields:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', ind1='<STR_LIT:C>', ind2='<STR_LIT:5>',<EOL>subfields=subfields)<EOL><DEDENT><DEDENT>if title:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', title)])<EOL><DEDENT>if date:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:c>', date),<EOL>('<STR_LIT:t>', '<STR_LIT>')])<EOL><DEDENT>if doi:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', ind1='<STR_LIT>', subfields=[('<STR_LIT:a>', doi),<EOL>('<STR_LIT:2>', '<STR_LIT>')])<EOL><DEDENT>if abstract:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', abstract),<EOL>('<STR_LIT>', '<STR_LIT>')])<EOL><DEDENT>first_author = True<EOL>for author in authors:<EOL><INDENT>if first_author:<EOL><INDENT>subfields = [('<STR_LIT:a>', author[<NUM_LIT:0>])]<EOL>if author[<NUM_LIT:1>]:<EOL><INDENT>subfields.append(('<STR_LIT:v>', author[<NUM_LIT:1>]))<EOL><DEDENT>record_add_field(rec, '<STR_LIT:100>', subfields=subfields)<EOL>first_author = False<EOL><DEDENT>else:<EOL><INDENT>subfields = [('<STR_LIT:a>', author[<NUM_LIT:0>])]<EOL>if author[<NUM_LIT:1>]:<EOL><INDENT>subfields.append(('<STR_LIT:v>', author[<NUM_LIT:1>]))<EOL><DEDENT>record_add_field(rec, '<STR_LIT>', subfields=subfields)<EOL><DEDENT><DEDENT>subfields = []<EOL>if journal and volume and first_page:<EOL><INDENT>subfields.append(('<STR_LIT:s>', "<STR_LIT>" % (journal,<EOL>volume,<EOL>first_page)))<EOL><DEDENT>if first_page and last_page:<EOL><INDENT>try:<EOL><INDENT>nuber_of_pages = int(last_page) - int(first_page)<EOL>record_add_field(rec, '<STR_LIT>',<EOL>subfields=[('<STR_LIT:a>', str(nuber_of_pages))])<EOL><DEDENT>except ValueError:<EOL><INDENT>pass<EOL><DEDENT>subfields.append(('<STR_LIT:c>', '<STR_LIT>' % (first_page,<EOL>last_page)))<EOL><DEDENT>if year:<EOL><INDENT>subfields.append(('<STR_LIT:y>', year))<EOL><DEDENT>record_add_field(rec, '<STR_LIT>', subfields=subfields)<EOL>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', '<STR_LIT>')])<EOL>if copyright_statement:<EOL><INDENT>record_add_field(rec, '<STR_LIT>',<EOL>subfields=[('<STR_LIT:f>', copyright_statement)])<EOL><DEDENT>if subject:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', ind1='<STR_LIT:1>', ind2='<STR_LIT>',<EOL>subfields=[('<STR_LIT:2>', '<STR_LIT>'),<EOL>('<STR_LIT:a>', subject)])<EOL><DEDENT>try:<EOL><INDENT>return record_xml_output(rec)<EOL><DEDENT>except UnicodeDecodeError:<EOL><INDENT>message = "<STR_LIT>" + doi<EOL>sys.stderr.write(message)<EOL>return "<STR_LIT>"<EOL><DEDENT>

Gets the Marc xml of the files in xaml_rich directory

:param fileName: the name of the file to parse.
:type fileName: string

:returns: a string with the marc xml version of the file.


def get_record(self, fileName, ref_extract_callback=None):

self.document = parse(fileName)<EOL>article_type = self._get_article_type()<EOL>if article_type not in ['<STR_LIT>',<EOL>'<STR_LIT>',<EOL>'<STR_LIT>']:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>rec = create_record()<EOL>title, subtitle, notes = self._get_title()<EOL>subfields = []<EOL>if subtitle:<EOL><INDENT>subfields.append(('<STR_LIT:b>', subtitle))<EOL><DEDENT>if title:<EOL><INDENT>subfields.append(('<STR_LIT:a>', title))<EOL>record_add_field(rec, '<STR_LIT>', subfields=subfields)<EOL><DEDENT>subjects = self.document.getElementsByTagName('<STR_LIT>')<EOL>subjects = map(xml_to_text, subjects)<EOL>for note_id in notes:<EOL><INDENT>note = self._get_note(note_id)<EOL>if note:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', note)])<EOL><DEDENT><DEDENT>for subject in subjects:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', ind1='<STR_LIT:1>', ind2='<STR_LIT>',<EOL>subfields=[('<STR_LIT:2>', '<STR_LIT>'),<EOL>('<STR_LIT:a>', subject)])<EOL><DEDENT>keywords = self._get_keywords()<EOL>for keyword in keywords:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', ind1='<STR_LIT:1>', subfields=[('<STR_LIT:a>', keyword),<EOL>('<STR_LIT>', '<STR_LIT>')])<EOL><DEDENT>journal, volume, issue, year, date, doi, page,fpage, lpage = self._get_publication_information()<EOL>astronomy_journals = ['<STR_LIT>', '<STR_LIT>']<EOL>if journal in astronomy_journals:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', ind1='<STR_LIT:1>', ind2='<STR_LIT>',<EOL>subfields=[('<STR_LIT:2>', '<STR_LIT>'),<EOL>('<STR_LIT:a>', '<STR_LIT>')])<EOL><DEDENT>if date:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:c>', date),<EOL>('<STR_LIT:t>', '<STR_LIT>')])<EOL><DEDENT>if doi:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', ind1='<STR_LIT>', subfields=[('<STR_LIT:a>', doi),<EOL>('<STR_LIT:2>', '<STR_LIT>')])<EOL><DEDENT>abstract = self._get_abstract()<EOL>abstract = self._format_abstract(abstract)<EOL>if abstract:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', abstract),<EOL>('<STR_LIT>', '<STR_LIT>')])<EOL><DEDENT>license, license_type, license_url = self._get_license()<EOL>subfields = []<EOL>if license:<EOL><INDENT>subfields.append(('<STR_LIT:a>', license))<EOL><DEDENT>if license_url:<EOL><INDENT>subfields.append(('<STR_LIT:u>', license_url))<EOL><DEDENT>if subfields:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=subfields)<EOL><DEDENT>if license_type == '<STR_LIT>':<EOL><INDENT>self._attach_fulltext(rec, doi)<EOL><DEDENT>number_of_pages = self._get_page_count()<EOL>if number_of_pages:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', number_of_pages)])<EOL><DEDENT>c_holder, c_year, c_statement = self._get_copyright()<EOL>if c_holder and c_year:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:d>', c_holder),<EOL>('<STR_LIT:g>', c_year),<EOL>('<STR_LIT:e>', '<STR_LIT>')])<EOL><DEDENT>elif c_statement:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:f>', c_statement),<EOL>('<STR_LIT:e>', '<STR_LIT>')])<EOL><DEDENT>subfields = []<EOL>if journal:<EOL><INDENT>subfields.append(('<STR_LIT:p>', journal))<EOL><DEDENT>if issue:<EOL><INDENT>subfields.append(('<STR_LIT:n>', issue))<EOL><DEDENT>if volume:<EOL><INDENT>subfields.append(('<STR_LIT:v>', volume))<EOL><DEDENT>if fpage and lpage:<EOL><INDENT>subfields.append(('<STR_LIT:c>', '<STR_LIT>' % (fpage,<EOL>lpage)))<EOL><DEDENT>elif page:<EOL><INDENT>subfields.append(('<STR_LIT:c>', page))<EOL><DEDENT>if year:<EOL><INDENT>subfields.append(('<STR_LIT:y>', year))<EOL><DEDENT>record_add_field(rec, '<STR_LIT>', subfields=subfields)<EOL>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', '<STR_LIT>')])<EOL>conference = '<STR_LIT>'<EOL>for tag in self.document.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>conference = xml_to_text(tag)<EOL><DEDENT>if conference:<EOL><INDENT>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', '<STR_LIT>')])<EOL>record_add_field(rec, '<STR_LIT>', subfields=[('<STR_LIT:a>', conference)])<EOL><DEDENT>self._add_references(rec, ref_extract_callback)<EOL>self._add_authors(rec)<EOL>try:<EOL><INDENT>return record_xml_output(rec)<EOL><DEDENT>except UnicodeDecodeError:<EOL><INDENT>message = "<STR_LIT>" + doi<EOL>sys.stderr.write(message)<EOL>return "<STR_LIT>"<EOL><DEDENT>

Gets the Marc xml of the files in xaml_jp directory

:param fileName: the name of the file to parse.
:type fileName: string
:param refextract_callback: callback to be used to extract
                            unstructured references. It should
                            return a marcxml formated string
                            of the reference.
:type refextract_callback: callable

:returns: a string with the marc xml version of the file.


label = get_value_in_tag(ref, '<STR_LIT:label>')<EOL>label = re.sub('<STR_LIT>', '<STR_LIT>', label)<EOL>for innerref in ref.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>ref_type = innerref.getAttribute('<STR_LIT>')<EOL>institution = get_value_in_tag(innerref, '<STR_LIT>')<EOL>report_no = '<STR_LIT>'<EOL>for tag in innerref.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>if tag.getAttribute('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>if tag.hasChildNodes():<EOL><INDENT>report_no = get_all_text(tag)<EOL><DEDENT><DEDENT><DEDENT>doi = '<STR_LIT>'<EOL>for tag in innerref.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>if tag.getAttribute('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>doi = xml_to_text(tag)<EOL><DEDENT><DEDENT>collaboration = get_value_in_tag(innerref, '<STR_LIT>')<EOL>authors = []<EOL>person_groups = innerref.getElementsByTagName('<STR_LIT>')<EOL>for author_group in person_groups:<EOL><INDENT>if author_group.getAttribute('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>for author in author_group.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>if author.hasChildNodes():<EOL><INDENT>authors.append(get_all_text(author))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>editors = []<EOL>for editor_group in person_groups:<EOL><INDENT>if editor_group.getAttribute('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>for editor in editor_group.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>if editor.hasChildNodes():<EOL><INDENT>editors.append(get_all_text(editor))<EOL><DEDENT><DEDENT><DEDENT><DEDENT>journal = get_value_in_tag(innerref, '<STR_LIT:source>')<EOL>journal, volume = fix_journal_name(journal, self.journal_mappings)<EOL>volume += get_value_in_tag(innerref, '<STR_LIT>')<EOL>if journal == '<STR_LIT>' or journal == '<STR_LIT>':<EOL><INDENT>issue = get_value_in_tag(innerref, '<STR_LIT>')<EOL>volume = volume[<NUM_LIT:2>:] + issue<EOL>journal = '<STR_LIT>'<EOL><DEDENT>page = get_value_in_tag(innerref, '<STR_LIT>')<EOL>year = get_value_in_tag(innerref, '<STR_LIT>')<EOL>external_link = get_value_in_tag(innerref, '<STR_LIT>')<EOL>arxiv = '<STR_LIT>'<EOL>for tag in innerref.getElementsByTagName('<STR_LIT>'):<EOL><INDENT>if tag.getAttribute('<STR_LIT>') == '<STR_LIT>':<EOL><INDENT>if tag.hasChildNodes():<EOL><INDENT>arxiv = get_all_text(tag)<EOL><DEDENT><DEDENT><DEDENT>arxiv = format_arxiv_id(arxiv)<EOL>publisher = get_value_in_tag(innerref, '<STR_LIT>')<EOL>publisher_location = get_value_in_tag(innerref, '<STR_LIT>')<EOL>if publisher_location:<EOL><INDENT>publisher = publisher_location + '<STR_LIT>' + publisher<EOL><DEDENT>unstructured_text = []<EOL>for child in innerref.childNodes:<EOL><INDENT>if child.nodeType == child.TEXT_NODE:<EOL><INDENT>text = child.nodeValue.strip()<EOL>text = re.sub(r'<STR_LIT>', '<STR_LIT>', text).strip()<EOL>if text.startswith('<STR_LIT:U+002C>'):<EOL><INDENT>text = text[<NUM_LIT:1>:].strip()<EOL><DEDENT>if text.endswith('<STR_LIT>'):<EOL><INDENT>text = institution + "<STR_LIT:U+0020>" + text<EOL>institution = '<STR_LIT>'<EOL>text = text.strip()<EOL><DEDENT>elif text.endswith('<STR_LIT>'):<EOL><INDENT>text += '<STR_LIT:.>'<EOL><DEDENT>elif text.endswith('<STR_LIT>'):<EOL><INDENT>if institution:<EOL><INDENT>text += '<STR_LIT:U+0020>' + institution<EOL>institution = '<STR_LIT>'<EOL><DEDENT>else:<EOL><INDENT>text = text[:-<NUM_LIT:1>]<EOL><DEDENT><DEDENT>elif text.startswith('<STR_LIT>'):<EOL><INDENT>article_title = get_value_in_tag(innerref, '<STR_LIT>')<EOL>text = institution + "<STR_LIT>" + article_title + "<STR_LIT>"<EOL>institution = '<STR_LIT>'<EOL><DEDENT>elif text == u'<STR_LIT>':<EOL><INDENT>text = '<STR_LIT>'<EOL><DEDENT>ignore_text = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>if text.startswith('<STR_LIT>'):<EOL><INDENT>temp = re.sub(r'<STR_LIT>', '<STR_LIT>', text)<EOL>if temp:<EOL><INDENT>volume += temp<EOL><DEDENT><DEDENT>elif len(text) > <NUM_LIT:1> and text not in ignore_textand not (text.isdigit() or text[:-<NUM_LIT:1>].isdigit()):<EOL><INDENT>unstructured_text.append(text)<EOL><DEDENT><DEDENT><DEDENT>if unstructured_text:<EOL><INDENT>unstructured_text = "<STR_LIT:U+0020>".join(unstructured_text)<EOL><DEDENT>if ref_type == '<STR_LIT>':<EOL><INDENT>if volume and not volume.lower().startswith('<STR_LIT>'):<EOL><INDENT>volume = '<STR_LIT>' + volume<EOL><DEDENT>if volume and page:<EOL><INDENT>volume = volume + '<STR_LIT>' + page<EOL><DEDENT><DEDENT>yield ref_type, doi, authors, collaboration, journal, volume, page, year,label, arxiv, publisher, institution, unstructured_text, external_link,report_no, editors<EOL><DEDENT>