if not os.path.isdir(self.outdir):<EOL><INDENT>os.makedirs(self.outdir)<EOL><DEDENT>if not os.path.isdir(self.resource_directory):<EOL><INDENT>os.makedirs(self.resource_directory)<EOL><DEDENT>data = {}<EOL>for input_file in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', input_file)<EOL>timestamp_format = None<EOL>with open(input_file, '<STR_LIT:r>') as infile:<EOL><INDENT>cpus = self.find_header(infile)<EOL>if len(cpus) == <NUM_LIT:0>:  <EOL><INDENT>logger.error("<STR_LIT>", input_file)<EOL>continue<EOL><DEDENT>prev_data = None    <EOL>curr_data = {}      <EOL>eth_data = {}<EOL>for line in infile:<EOL><INDENT>if self.is_header_line(line):  <EOL><INDENT>prev_data = curr_data<EOL>curr_data = {}<EOL>for eth in eth_data:<EOL><INDENT>outcsv = self.get_csv('<STR_LIT>', eth)<EOL>if outcsv not in data:<EOL><INDENT>data[outcsv] = []<EOL><DEDENT>data[outcsv].append(ts + '<STR_LIT:U+002C>' + str(eth_data[eth]))<EOL><DEDENT>eth_data = {}<EOL>continue<EOL><DEDENT>words = line.split()<EOL>if len(words) <= <NUM_LIT:4>:  <EOL><INDENT>continue<EOL><DEDENT>ts = words[<NUM_LIT:0>] + "<STR_LIT:U+0020>" + words[<NUM_LIT:1>]<EOL>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(ts)<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(ts, timestamp_format)<EOL>if self.ts_out_of_range(ts):  <EOL><INDENT>continue<EOL><DEDENT>device = words[<NUM_LIT:2>].strip('<STR_LIT::>')  <EOL>if re.match("<STR_LIT>", device):<EOL><INDENT>if (<NUM_LIT:4> + len(cpus)) < len(words):<EOL><INDENT>device = words[<NUM_LIT:4> + len(cpus)] + "<STR_LIT>" + device<EOL><DEDENT>else:<EOL><INDENT>device = "<STR_LIT>" + device<EOL><DEDENT><DEDENT>else:<EOL><INDENT>device = "<STR_LIT:->".join(words[(<NUM_LIT:3> + len(cpus)):]) + "<STR_LIT>" + device<EOL><DEDENT>for (cpu, datum) in zip(cpus, words[<NUM_LIT:3>:]):<EOL><INDENT>if self.CPUS and cpu not in self.CPUS:  <EOL><INDENT>continue<EOL><DEDENT>outcsv = self.get_csv(cpu, device)<EOL>curr_data[outcsv] = int(datum)<EOL>if outcsv in data:<EOL><INDENT>datum = int(datum) - prev_data[outcsv]  <EOL><DEDENT>else:<EOL><INDENT>data[outcsv] = []<EOL>datum = <NUM_LIT:0>  <EOL><DEDENT>data[outcsv].append(ts + '<STR_LIT:U+002C>' + str(datum))<EOL>m = re.search("<STR_LIT>", device)<EOL>if m:<EOL><INDENT>eth = m.group('<STR_LIT>')<EOL>if eth not in eth_data:<EOL><INDENT>eth_data[eth] = <NUM_LIT:0><EOL><DEDENT>eth_data[eth] += datum<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>for csv in data.keys():<EOL><INDENT>self.csv_files.append(csv)<EOL>with open(csv, '<STR_LIT:w>') as csvf:<EOL><INDENT>csvf.write('<STR_LIT:\n>'.join(sorted(data[csv])))<EOL><DEDENT><DEDENT>return True<EOL>

Processes the files for each IRQ and each CPU in terms of the differences.
Also produces accumulated interrupt count differences for each set of Ethernet IRQs.
Generally Ethernet has 8 TxRx IRQs thus all are combined so that one can see the overall interrupts being generated by the NIC.

Simplified Interrupt File Format: (See examples for example log)

                                    CPU0   CPU1
  2014-10-29 00:27:42.15161    59:    29      2    IR-IO-APIC-edge    timer
  2014-10-29 00:27:42.15161    60:  2123      0    IR-PCI-MSI-edge    eth0

                                    CPU0   CPU1
  2014-10-29 00:27:42.15161    59:    29      2    IR-IO-APIC-edge    timer
  2014-10-29 00:27:42.15161    60:  2123      0    IR-PCI-MSI-edge    eth0

:returns: True or False whether parsing was successful or not.


file_status = True<EOL>for input_file in self.infile_list:<EOL><INDENT>file_status = file_status and naarad.utils.is_valid_file(input_file)<EOL>if not file_status:<EOL><INDENT>return False<EOL><DEDENT><DEDENT>status = True<EOL>cur_zone = None<EOL>cur_submetric = None<EOL>cur_value = None<EOL>data = {}  <EOL>for input_file in self.infile_list:<EOL><INDENT>logger.info('<STR_LIT>', input_file)<EOL>timestamp_format = None<EOL>with open(input_file) as fh:<EOL><INDENT>for line in fh:<EOL><INDENT>words = line.replace('<STR_LIT:U+002C>', '<STR_LIT:U+0020>').split()           <EOL>if len(words) < <NUM_LIT:3>:<EOL><INDENT>continue<EOL><DEDENT>ts = words[<NUM_LIT:0>] + "<STR_LIT:U+0020>" + words[<NUM_LIT:1>]<EOL>if not timestamp_format or timestamp_format == '<STR_LIT>':<EOL><INDENT>timestamp_format = naarad.utils.detect_timestamp_format(ts)<EOL><DEDENT>if timestamp_format == '<STR_LIT>':<EOL><INDENT>continue<EOL><DEDENT>ts = naarad.utils.get_standardized_timestamp(ts, timestamp_format)<EOL>if self.ts_out_of_range(ts):<EOL><INDENT>continue<EOL><DEDENT>if words[<NUM_LIT:2>] == '<STR_LIT>':  <EOL><INDENT>cols = words[<NUM_LIT:2>:]<EOL>cur_zone = '<STR_LIT:.>'.join(cols)<EOL>continue<EOL><DEDENT>elif words[<NUM_LIT:2>] == '<STR_LIT>':  <EOL><INDENT>cur_submetric = words[<NUM_LIT:2>] + '<STR_LIT:.>' + words[<NUM_LIT:3>]  <EOL>cur_value = words[<NUM_LIT:4>]<EOL><DEDENT>elif words[<NUM_LIT:2>] in self.processed_sub_metrics:<EOL><INDENT>cur_submetric = '<STR_LIT>' + '<STR_LIT:.>' + words[<NUM_LIT:2>]  <EOL>cur_value = words[<NUM_LIT:3>]<EOL><DEDENT>elif words[<NUM_LIT:2>] in self.skipped_sub_metrics:<EOL><INDENT>continue<EOL><DEDENT>else:   <EOL><INDENT>cur_submetric = words[<NUM_LIT:2>]<EOL>cur_value = words[<NUM_LIT:3>]<EOL><DEDENT>col = cur_zone + '<STR_LIT:.>' + cur_submetric  <EOL>if cur_zone and self.zones and cur_zone not in self.zones:<EOL><INDENT>continue<EOL><DEDENT>self.sub_metric_unit[col] = '<STR_LIT>'  <EOL>if self.sub_metrics and cur_submetric and cur_submetric not in self.sub_metrics:<EOL><INDENT>continue<EOL><DEDENT>if col in self.column_csv_map:<EOL><INDENT>out_csv = self.column_csv_map[col]<EOL><DEDENT>else:<EOL><INDENT>out_csv = self.get_csv(col)   <EOL>data[out_csv] = []<EOL><DEDENT>data[out_csv].append(ts + "<STR_LIT:U+002C>" + cur_value)<EOL><DEDENT><DEDENT><DEDENT>for csv in data.keys():<EOL><INDENT>self.csv_files.append(csv)<EOL>with open(csv, '<STR_LIT:w>') as fh:<EOL><INDENT>fh.write('<STR_LIT:\n>'.join(sorted(data[csv])))<EOL><DEDENT><DEDENT>return status<EOL>

Parse the vmstat file
:return: status of the metric parse


for metric_type in self.supported_sar_types:<EOL><INDENT>if metric_type in metric_name:<EOL><INDENT>return metric_type<EOL><DEDENT><DEDENT>logger.error('<STR_LIT>'<EOL>'<STR_LIT>', metric_name, self.supported_sar_types)<EOL>return '<STR_LIT>'<EOL>

Method to extract SAR metric names from the section given in the config. The SARMetric class assumes that
the section name will contain the SAR types listed in self.supported_sar_types tuple

:param str metric_name: Section name from the config
:return: str which identifies what kind of SAR metric the section represents


self._default_test_id += <NUM_LIT:1><EOL>self._analyses[self._default_test_id] = _Analysis(ts_start=None, config=config, test_id=self._default_test_id)<EOL>

Create Analysis and save in Naarad from config
:param config:
:return:


failed_analyses = []<EOL>for test_id in self._analyses.keys():<EOL><INDENT>if self._analyses[test_id].status != CONSTANTS.OK:<EOL><INDENT>failed_analyses.append(test_id)<EOL><DEDENT><DEDENT>return failed_analyses<EOL>

Returns a list of test_id for which naarad analysis failed
:return: list of test_ids


try:<EOL><INDENT>os.makedirs(analysis.output_directory)<EOL><DEDENT>except OSError as exception:<EOL><INDENT>if exception.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>try:<EOL><INDENT>resource_directory = os.path.join(analysis.output_directory, analysis.resource_path)<EOL>os.makedirs(resource_directory)<EOL><DEDENT>except OSError as exception:<EOL><INDENT>if exception.errno != errno.EEXIST:<EOL><INDENT>raise<EOL><DEDENT><DEDENT>

Create the necessary output and resource directories for the specified analysis
:param: analysis: analysis associated with a given test_id


def _run_pre(self, analysis, run_steps):

workload_run_steps = []<EOL>for run_step in sorted(run_steps, key=lambda step: step.run_rank):<EOL><INDENT>run_step.run()<EOL>if run_step.run_type == CONSTANTS.RUN_TYPE_WORKLOAD:<EOL><INDENT>workload_run_steps.append(run_step)<EOL><DEDENT><DEDENT>if len(workload_run_steps) > <NUM_LIT:0>:<EOL><INDENT>analysis.ts_start, analysis.ts_end = naarad.utils.get_run_time_period(workload_run_steps)<EOL><DEDENT>return CONSTANTS.OK<EOL>

If Naarad is run in CLI mode, execute any pre run steps specified in the config. ts_start/ts_end are set based on
workload run steps if any.
:param: analysis: The analysis object being processed
:param: run_steps: list of post run steps


for run_step in sorted(run_steps, key=lambda step: step.run_rank):<EOL><INDENT>run_step.run()<EOL><DEDENT>return CONSTANTS.OK<EOL>

If Naarad is run in CLI mode, execute any post run steps specified in the config
:param: run_steps: list of post run steps


def _process_args(self, analysis, args):

if args.exit_code:<EOL><INDENT>self.return_exit_code = args.exit_code<EOL><DEDENT>if args.no_plots:<EOL><INDENT>self.skip_plots = args.no_plots<EOL><DEDENT>if args.start:<EOL><INDENT>analysis.ts_start = naarad.utils.get_standardized_timestamp(args.start, None)<EOL><DEDENT>if args.end:<EOL><INDENT>analysis.ts_end = naarad.utils.get_standardized_timestamp(args.end, None)<EOL><DEDENT>if args.variables:<EOL><INDENT>analysis.variables = naarad.utils.get_variables(args)<EOL><DEDENT>return CONSTANTS.OK<EOL>

When Naarad is run in CLI mode, get the CL arguments and update the analysis
:param: analysis: The analysis being processed
:param: args: Command Line Arguments received by naarad


def analyze(self, input_directory, output_directory, **kwargs):

is_api_call = True<EOL>if len(self._analyses) == <NUM_LIT:0>:<EOL><INDENT>if '<STR_LIT>' not in kwargs.keys():<EOL><INDENT>return CONSTANTS.ERROR<EOL><DEDENT>self.create_analysis(kwargs['<STR_LIT>'])<EOL><DEDENT>if '<STR_LIT:args>' in kwargs:<EOL><INDENT>self._process_args(self._analyses[<NUM_LIT:0>], kwargs['<STR_LIT:args>'])<EOL>is_api_call = False<EOL><DEDENT>error_count = <NUM_LIT:0><EOL>self._input_directory = input_directory<EOL>self._output_directory = output_directory<EOL>for test_id in sorted(self._analyses.keys()):<EOL><INDENT>if not self._analyses[test_id].input_directory:<EOL><INDENT>self._analyses[test_id].input_directory = input_directory<EOL><DEDENT>if not self._analyses[test_id].output_directory:<EOL><INDENT>if len(self._analyses) > <NUM_LIT:1>:<EOL><INDENT>self._analyses[test_id].output_directory = os.path.join(output_directory, str(test_id))<EOL><DEDENT>else:<EOL><INDENT>self._analyses[test_id].output_directory = output_directory<EOL><DEDENT><DEDENT>if('<STR_LIT>' in kwargs.keys()) and (not self._analyses[test_id].config):<EOL><INDENT>self._analyses[test_id].config = kwargs['<STR_LIT>']<EOL><DEDENT>self._create_output_directories(self._analyses[test_id])<EOL>self._analyses[test_id].status = self.run(self._analyses[test_id], is_api_call, **kwargs)<EOL>if self._analyses[test_id].status != CONSTANTS.OK:<EOL><INDENT>error_count += <NUM_LIT:1><EOL><DEDENT><DEDENT>if len(self._analyses) == <NUM_LIT:1>:<EOL><INDENT>return self._analyses[<NUM_LIT:0>].status<EOL><DEDENT>elif error_count > <NUM_LIT:0>:<EOL><INDENT>return CONSTANTS.ERROR<EOL><DEDENT>else:<EOL><INDENT>return CONSTANTS.OK<EOL><DEDENT>

Run all the analysis saved in self._analyses, sorted by test_id.
This is useful when Naarad() is used by other programs and multiple analyses are run
In naarad CLI mode, len(_analyses) == 1
:param: input_directory: location of log files
:param: output_directory: root directory for analysis output
:param: **kwargs: Optional keyword args
:return: int: status code.


def run(self, analysis, is_api_call, **kwargs):

threads = []<EOL>crossplots = []<EOL>report_args = {}<EOL>metrics = defaultdict()<EOL>run_steps = defaultdict(list)<EOL>discovery_mode = False<EOL>graph_timezone = None<EOL>graphing_library = None<EOL>if isinstance(analysis.config, str):<EOL><INDENT>if not naarad.utils.is_valid_file(analysis.config):<EOL><INDENT>return CONSTANTS.INVALID_CONFIG<EOL><DEDENT>config_object = ConfigParser.ConfigParser(analysis.variables)<EOL>config_object.optionxform = str<EOL>config_object.read(analysis.config)<EOL><DEDENT>elif isinstance(analysis.config, ConfigParser.ConfigParser):<EOL><INDENT>config_object = analysis.config<EOL><DEDENT>else:<EOL><INDENT>if is_api_call:<EOL><INDENT>return CONSTANTS.INVALID_CONFIG<EOL><DEDENT>else:<EOL><INDENT>metrics['<STR_LIT>'] = naarad.utils.discover_by_name(analysis.input_directory, analysis.output_directory)<EOL>if len(metrics['<STR_LIT>']) == <NUM_LIT:0>:<EOL><INDENT>logger.warning('<STR_LIT>', analysis.input_directory)<EOL>return CONSTANTS.ERROR<EOL><DEDENT>else:<EOL><INDENT>discovery_mode = True<EOL>metrics['<STR_LIT>'] = []<EOL><DEDENT><DEDENT><DEDENT>if not discovery_mode:<EOL><INDENT>metrics, run_steps, crossplots, report_args, graph_timezone, graphing_library = self._process_naarad_config(config_object, analysis)<EOL><DEDENT>if graphing_library is None:<EOL><INDENT>graphing_library = CONSTANTS.DEFAULT_GRAPHING_LIBRARY<EOL><DEDENT>if graphing_library not in self.available_graphing_modules.keys():<EOL><INDENT>logger.error("<STR_LIT>", graphing_library)<EOL>self.skip_plots = True<EOL><DEDENT>if not is_api_call:<EOL><INDENT>self._run_pre(analysis, run_steps['<STR_LIT>'])<EOL><DEDENT>for metric in metrics['<STR_LIT>']:<EOL><INDENT>if analysis.ts_start:<EOL><INDENT>metric.ts_start = analysis.ts_start<EOL><DEDENT>if analysis.ts_end:<EOL><INDENT>metric.ts_end = analysis.ts_end<EOL><DEDENT>thread = threading.Thread(target=naarad.utils.parse_and_plot_single_metrics,<EOL>args=(metric, graph_timezone, analysis.output_directory, analysis.input_directory, graphing_library, self.skip_plots))<EOL>thread.start()<EOL>threads.append(thread)<EOL><DEDENT>for t in threads:<EOL><INDENT>t.join()<EOL><DEDENT>for metric in metrics['<STR_LIT>']:<EOL><INDENT>thread = threading.Thread(target=naarad.utils.parse_and_plot_single_metrics,<EOL>args=(metric, graph_timezone, analysis.output_directory, analysis.input_directory, graphing_library, self.skip_plots))<EOL>thread.start()<EOL>threads.append(thread)<EOL><DEDENT>for t in threads:<EOL><INDENT>t.join()<EOL><DEDENT>self._set_sla_data(analysis.test_id, metrics['<STR_LIT>'] + metrics['<STR_LIT>'])<EOL>self._set_stats_data(analysis.test_id, metrics['<STR_LIT>'] + metrics['<STR_LIT>'])<EOL>if len(crossplots) > <NUM_LIT:0> and not self.skip_plots:<EOL><INDENT>correlated_plots = naarad.utils.nway_plotting(crossplots, metrics['<STR_LIT>'] + metrics['<STR_LIT>'],<EOL>os.path.join(analysis.output_directory, analysis.resource_path),<EOL>analysis.resource_path, graphing_library)<EOL><DEDENT>else:<EOL><INDENT>correlated_plots = []<EOL><DEDENT>rpt = reporting_modules['<STR_LIT>'](None, analysis.output_directory, os.path.join(analysis.output_directory, analysis.resource_path), analysis.resource_path,<EOL>metrics['<STR_LIT>'] + metrics['<STR_LIT>'], correlated_plots=correlated_plots, **report_args)<EOL>rpt.generate()<EOL>if not is_api_call:<EOL><INDENT>self._run_post(run_steps['<STR_LIT>'])<EOL><DEDENT>if self.return_exit_code:<EOL><INDENT>for metric in metrics['<STR_LIT>'] + metrics['<STR_LIT>']:<EOL><INDENT>if metric.status == CONSTANTS.SLA_FAILED:<EOL><INDENT>return CONSTANTS.SLA_FAILURE<EOL><DEDENT><DEDENT><DEDENT>return CONSTANTS.OK<EOL>

:param analysis: Run naarad analysis for the specified analysis object
:param **kwargs: Additional keyword args can be passed in here for future enhancements
:return:


def diff_reports_by_location(self, report1_location, report2_location, output_directory, config=None, **kwargs):

if kwargs:<EOL><INDENT>if '<STR_LIT>' in kwargs.keys():<EOL><INDENT>output_directory = kwargs['<STR_LIT>']<EOL><DEDENT><DEDENT>diff_report = Diff([NaaradReport(report1_location, None), NaaradReport(report2_location, None)], '<STR_LIT>',<EOL>output_directory, os.path.join(output_directory, self._resource_path), self._resource_path)<EOL>if config:<EOL><INDENT>naarad.utils.extract_diff_sla_from_config_file(diff_report, config)<EOL><DEDENT>diff_report.generate()<EOL>if diff_report.sla_failures > <NUM_LIT:0>:<EOL><INDENT>return CONSTANTS.SLA_FAILURE<EOL><DEDENT>if diff_report.status != '<STR_LIT:OK>':<EOL><INDENT>return CONSTANTS.ERROR<EOL><DEDENT>return CONSTANTS.OK<EOL>

Create a diff report using report1 as a baseline
:param: report1_location: report to be used as baseline
:param: report2_location: report to compare against baseline
:param: config file for diff (optional)
:param: **kwargs: keyword arguments


def _process_naarad_config(self, config, analysis):

graph_timezone = None<EOL>output_directory = analysis.output_directory<EOL>resource_path = analysis.resource_path<EOL>run_steps = defaultdict(list)<EOL>metrics = defaultdict(list)<EOL>indir_default = '<STR_LIT>'<EOL>crossplots = []<EOL>report_args = {}<EOL>graphing_library = None<EOL>ts_start, ts_end = None, None<EOL>if config.has_section('<STR_LIT>'):<EOL><INDENT>ts_start, ts_end = naarad.utils.parse_global_section(config, '<STR_LIT>')<EOL>if config.has_option('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>naarad.utils.parse_user_defined_metric_classes(config, metric_classes)<EOL><DEDENT>config.remove_section('<STR_LIT>')<EOL><DEDENT>if config.has_section('<STR_LIT>'):<EOL><INDENT>report_args = naarad.utils.parse_report_section(config, '<STR_LIT>')<EOL>config.remove_section('<STR_LIT>')<EOL><DEDENT>for section in config.sections():<EOL><INDENT>if section == '<STR_LIT>':<EOL><INDENT>graphing_library, crossplots, outdir_default, indir_default, graph_timezone =naarad.utils.parse_graph_section(config, section, output_directory, indir_default)<EOL><DEDENT>elif section.startswith('<STR_LIT>'):<EOL><INDENT>run_step = naarad.utils.parse_run_step_section(config, section)<EOL>if not run_step:<EOL><INDENT>logger.error('<STR_LIT>', section)<EOL>continue<EOL><DEDENT>if run_step.run_order == CONSTANTS.PRE_ANALYSIS_RUN:<EOL><INDENT>run_steps['<STR_LIT>'].append(run_step)<EOL><DEDENT>elif run_step.run_order == CONSTANTS.DURING_ANALYSIS_RUN:<EOL><INDENT>run_steps['<STR_LIT>'].append(run_step)<EOL><DEDENT>elif run_step.run_order == CONSTANTS.POST_ANALYSIS_RUN:<EOL><INDENT>run_steps['<STR_LIT>'].append(run_step)<EOL><DEDENT>else:<EOL><INDENT>logger.error('<STR_LIT>')<EOL><DEDENT><DEDENT>else:<EOL><INDENT>if not naarad.utils.is_valid_metric_name(section):<EOL><INDENT>logger.critical('<STR_LIT>'<EOL>% section)<EOL>return CONSTANTS.CRITICAL_FAILURE<EOL><DEDENT>if section == '<STR_LIT>':<EOL><INDENT>hostname, infile, label, ts_start, ts_end, precision, kwargs, rule_strings =naarad.utils.parse_basic_metric_options(config, section)<EOL>sar_metrics = naarad.utils.get_all_sar_objects(metrics, infile, hostname, output_directory, label, ts_start,<EOL>ts_end, None)<EOL>for sar_metric in sar_metrics:<EOL><INDENT>if sar_metric.ts_start is None and (sar_metric.ts_end is None or sar_metric.ts_end > ts_start):<EOL><INDENT>sar_metric.ts_start = ts_start<EOL><DEDENT>if sar_metric.ts_end is None and (sar_metric.ts_start is None or ts_end > sar_metric.ts_start):<EOL><INDENT>sar_metric.ts_end = ts_end<EOL><DEDENT><DEDENT>metrics['<STR_LIT>'].extend(sar_metrics)<EOL><DEDENT>else:<EOL><INDENT>new_metric = naarad.utils.parse_metric_section(config, section, metric_classes, metrics['<STR_LIT>'],<EOL>aggregate_metric_classes, output_directory, resource_path)<EOL>if new_metric.ts_start is None and (new_metric.ts_end is None or new_metric.ts_end > ts_start):<EOL><INDENT>new_metric.ts_start = ts_start<EOL><DEDENT>if new_metric.ts_end is None and (new_metric.ts_start is None or ts_end > new_metric.ts_start):<EOL><INDENT>new_metric.ts_end = ts_end<EOL><DEDENT>metric_type = section.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>if metric_type in aggregate_metric_classes:<EOL><INDENT>metrics['<STR_LIT>'].append(new_metric)<EOL><DEDENT>else:<EOL><INDENT>metrics['<STR_LIT>'].append(new_metric)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return metrics, run_steps, crossplots, report_args, graph_timezone, graphing_library<EOL>

Process the config file associated with a particular analysis and return metrics, run_steps and crossplots.
Also sets output directory and resource_path for an anlaysis


user_defined_metric_list = config_obj.get('<STR_LIT>', '<STR_LIT>').split()<EOL>for udm_string in user_defined_metric_list:<EOL><INDENT>try:<EOL><INDENT>metric_name, metric_class_name, metric_file = udm_string.split('<STR_LIT::>')<EOL><DEDENT>except ValueError:<EOL><INDENT>logger.error('<STR_LIT>')<EOL>continue<EOL><DEDENT>module_name = os.path.splitext(os.path.basename(metric_file))[<NUM_LIT:0>]<EOL>try:<EOL><INDENT>new_module = imp.load_source(module_name, metric_file)<EOL>new_class = getattr(new_module, metric_class_name)<EOL>if metric_name in metric_classes.keys():<EOL><INDENT>logger.warn('<STR_LIT>', metric_name)<EOL><DEDENT>metric_classes[metric_name] = new_class<EOL><DEDENT>except ImportError:<EOL><INDENT>logger.error('<STR_LIT>', metric_name)<EOL>continue<EOL><DEDENT><DEDENT>

Parse the user defined metric class information
:param config_obj: ConfigParser object
:param metric_classes: list of metric classes to be updated
:return:


regex = re.compile(r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>'<EOL>r'<STR_LIT>', re.IGNORECASE)<EOL>if regex.match(url):<EOL><INDENT>logger.info("<STR_LIT>")<EOL>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

Check if a given string is in the correct URL format or not

:param str url:
:return: True or False


try:<EOL><INDENT>(local_file, headers) = urllib.urlretrieve(url)<EOL><DEDENT>except:<EOL><INDENT>sys.exit("<STR_LIT>" + url + "<STR_LIT>")<EOL><DEDENT>return local_file<EOL>

Download a file pointed to by url to a temp file on local disk

:param str url:
:return: local_file


reg = re.compile('<STR_LIT>')<EOL>if reg.match(metric_name) and not metric_name.startswith('<STR_LIT:.>'):<EOL><INDENT>return True<EOL><DEDENT>else:<EOL><INDENT>return False<EOL><DEDENT>

check the validity of metric_name in config; the metric_name will be used for creation of sub-dir, so only contains: alphabet, digits , '.', '-' and '_'
:param str metric_name: metric_name
:return: True if valid


init_ts_start = get_standardized_timestamp('<STR_LIT>', None)<EOL>ts_start = init_ts_start<EOL>ts_end = '<STR_LIT:0>'<EOL>for run_step in run_steps:<EOL><INDENT>if run_step.ts_start and run_step.ts_end:<EOL><INDENT>if run_step.ts_start < ts_start:<EOL><INDENT>ts_start = run_step.ts_start<EOL><DEDENT>if run_step.ts_end > ts_end:<EOL><INDENT>ts_end = run_step.ts_end<EOL><DEDENT><DEDENT><DEDENT>if ts_end == '<STR_LIT:0>':<EOL><INDENT>ts_end = None<EOL><DEDENT>if ts_start == init_ts_start:<EOL><INDENT>ts_start = None<EOL><DEDENT>logger.info('<STR_LIT>' + str(ts_start) + '<STR_LIT>' + str(ts_end))<EOL>return ts_start, ts_end<EOL>

This method finds the time range which covers all the Run_Steps

:param run_steps: list of Run_Step objects
:return: tuple of start and end timestamps


rule_strings = {}<EOL>kwargs = dict(config_obj.items(section))<EOL>for key in kwargs.keys():<EOL><INDENT>if key.endswith('<STR_LIT>'):<EOL><INDENT>rule_strings[key.replace('<STR_LIT>', '<STR_LIT>')] = kwargs[key]<EOL>del kwargs[key]<EOL><DEDENT><DEDENT>return rule_strings, kwargs<EOL>

Extract rule strings from a section
:param config_obj: ConfigParser object
:param section: Section name
:return: the rule strings


rule_strings = {}<EOL>config_obj = ConfigParser.ConfigParser()<EOL>config_obj.optionxform = str<EOL>config_obj.read(options_file)<EOL>for section in config_obj.sections():<EOL><INDENT>rule_strings, kwargs = get_rule_strings(config_obj, section)<EOL>for (key, val) in rule_strings.iteritems():<EOL><INDENT>set_sla(obj, section, key, val)<EOL><DEDENT><DEDENT>

Helper function to parse diff config file, which contains SLA rules for diff comparisons


infile = {}<EOL>aggr_hosts = None<EOL>aggr_metrics = None<EOL>ts_start = None<EOL>ts_end = None<EOL>precision = None<EOL>hostname = "<STR_LIT:localhost>"<EOL>rule_strings = {}<EOL>important_sub_metrics = None<EOL>anomaly_detection_metrics = None<EOL>try:<EOL><INDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>important_sub_metrics = config_obj.get(section, '<STR_LIT>').split()<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>hostname = config_obj.get(section, '<STR_LIT>')<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>infile = config_obj.get(section, '<STR_LIT>').split()<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>label = sanitize_string_section_name(section)<EOL>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>ts_start = get_standardized_timestamp(config_obj.get(section, '<STR_LIT>'), None)<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>ts_end = get_standardized_timestamp(config_obj.get(section, '<STR_LIT>'), None)<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>precision = config_obj.get(section, '<STR_LIT>')<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>aggr_hosts = config_obj.get(section, '<STR_LIT>')<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>aggr_metrics = config_obj.get(section, '<STR_LIT>')<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>anomaly_detection_metrics = config_obj.get(section, '<STR_LIT>').split()<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>rule_strings, other_options = get_rule_strings(config_obj, section)<EOL><DEDENT>except ConfigParser.NoOptionError:<EOL><INDENT>logger.exception("<STR_LIT>" + section)<EOL>sys.exit()<EOL><DEDENT>return (hostname, infile, aggr_hosts, aggr_metrics, label, ts_start, ts_end, precision, aggr_metrics, other_options,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics)<EOL>

Parse basic options from metric sections of the config
:param config_obj: ConfigParser object
:param section: Section name
:return: all the parsed options


def parse_metric_section(config_obj, section, metric_classes, metrics, aggregate_metric_classes, outdir_default, resource_path):

(hostname, infile, aggr_hosts, aggr_metrics, label, ts_start, ts_end, precision, aggr_metrics, other_options,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics) = parse_basic_metric_options(config_obj, section)<EOL>metric_type = section.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>if metric_type in aggregate_metric_classes:<EOL><INDENT>new_metric = initialize_aggregate_metric(section, aggr_hosts, aggr_metrics, metrics, outdir_default, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, other_options)<EOL><DEDENT>else:<EOL><INDENT>new_metric = initialize_metric(section, infile, hostname, aggr_metrics, outdir_default, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, other_options)<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT:ignore>') and config_obj.getint(section, '<STR_LIT:ignore>') == <NUM_LIT:1>:<EOL><INDENT>new_metric.ignore = True<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>new_metric.calc_metrics = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>new_metric.precision = precision<EOL>return new_metric<EOL>

Parse a metric section and create a Metric object
:param config_obj: ConfigParser object
:param section: Section name
:param metric_classes: List of valid metric types
:param metrics: List of all regular metric objects (used by aggregate metric)
:param aggregate_metric_classes: List of all valid aggregate metric types
:param outdir_default: Default output directory
:param resource_path: Default resource directory
:return: An initialized Metric object


ts_start = None<EOL>ts_end = None<EOL>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>ts_start = get_standardized_timestamp(config_obj.get(section, '<STR_LIT>'), None)<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>ts_end = get_standardized_timestamp(config_obj.get(section, '<STR_LIT>'), None)<EOL>config_obj.remove_option(section, '<STR_LIT>')<EOL><DEDENT>return ts_start, ts_end<EOL>

Parse GLOBAL section in the config to return global settings
:param config_obj: ConfigParser object
:param section: Section name
:return: ts_start and ts_end time


kill_after_seconds = None<EOL>try:<EOL><INDENT>run_cmd = config_obj.get(section, '<STR_LIT>')<EOL>run_rank = int(config_obj.get(section, '<STR_LIT>'))<EOL><DEDENT>except ConfigParser.NoOptionError:<EOL><INDENT>logger.exception("<STR_LIT>" + section)<EOL>sys.exit()<EOL><DEDENT>except ValueError:<EOL><INDENT>logger.error("<STR_LIT>", config_obj.get(section, '<STR_LIT>'), section)<EOL>sys.exit()<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>run_type = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>run_type = CONSTANTS.RUN_TYPE_WORKLOAD<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>run_order = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>run_order = CONSTANTS.PRE_ANALYSIS_RUN<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>call_type = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>call_type = '<STR_LIT>'<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>try:<EOL><INDENT>kill_after_seconds = int(config_obj.get(section, '<STR_LIT>'))<EOL><DEDENT>except ValueError:<EOL><INDENT>logger.error("<STR_LIT>", config_obj.get(section, '<STR_LIT>'), section)<EOL><DEDENT><DEDENT>if call_type == '<STR_LIT>':<EOL><INDENT>run_step_obj = Local_Cmd(run_type, run_cmd, call_type, run_order, run_rank, kill_after_seconds=kill_after_seconds)<EOL><DEDENT>else:<EOL><INDENT>logger.error('<STR_LIT>')<EOL>run_step_obj = None<EOL><DEDENT>return run_step_obj<EOL>

Parse a RUN-STEP section in the config to return a Run_Step object
:param config_obj: ConfigParser objection
:param section: Section name
:return: an initialized Run_Step object


def parse_graph_section(config_obj, section, outdir_default, indir_default):

graph_timezone = None<EOL>graphing_library = CONSTANTS.DEFAULT_GRAPHING_LIBRARY<EOL>crossplots = []<EOL>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>graphing_library = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>graphs_string = config_obj.get(section, '<STR_LIT>')<EOL>crossplots = graphs_string.split()<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>outdir_default = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>outdir_default = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>indir_default = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>graph_timezone = config_obj.get(section, '<STR_LIT>')<EOL>if graph_timezone not in ("<STR_LIT>", "<STR_LIT>", "<STR_LIT>"):<EOL><INDENT>logger.warn('<STR_LIT>' + graph_timezone + '<STR_LIT>')<EOL>graph_timezone = "<STR_LIT>"<EOL><DEDENT><DEDENT>return graphing_library, crossplots, outdir_default, indir_default, graph_timezone<EOL>

Parse the GRAPH section of the config to extract useful values
:param config_obj: ConfigParser object
:param section: Section name
:param outdir_default: Default output directory passed in args
:param indir_default: Default input directory passed in args
:return: List of options extracted from the GRAPH section


report_kwargs = {}<EOL>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>if config_obj.has_option(section, '<STR_LIT>'):<EOL><INDENT>report_kwargs['<STR_LIT>'] = config_obj.get(section, '<STR_LIT>')<EOL><DEDENT>return report_kwargs<EOL>

parse the [REPORT] section of a config file to extract various reporting options to be passed to the Report object
:param: config_obj : configparser object for the config file passed in to naarad
:param: section: name of the section. 'REPORT' should be passed in here
:return: report_kwargs: dictionary of Reporting options and values specified in config.


def calculate_stats(data_list, stats_to_calculate=['<STR_LIT>', '<STR_LIT>'], percentiles_to_calculate=[]):

stats_to_numpy_method_map = {<EOL>'<STR_LIT>': numpy.mean,<EOL>'<STR_LIT>': numpy.mean,<EOL>'<STR_LIT>': numpy.std,<EOL>'<STR_LIT>': numpy.std,<EOL>'<STR_LIT>': numpy.median,<EOL>'<STR_LIT>': numpy.amin,<EOL>'<STR_LIT>': numpy.amax<EOL>}<EOL>calculated_stats = {}<EOL>calculated_percentiles = {}<EOL>if len(data_list) == <NUM_LIT:0>:<EOL><INDENT>return calculated_stats, calculated_percentiles<EOL><DEDENT>for stat in stats_to_calculate:<EOL><INDENT>if stat in stats_to_numpy_method_map.keys():<EOL><INDENT>calculated_stats[stat] = stats_to_numpy_method_map[stat](data_list)<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>" + str(stat))<EOL><DEDENT><DEDENT>for percentile in percentiles_to_calculate:<EOL><INDENT>if isinstance(percentile, float) or isinstance(percentile, int):<EOL><INDENT>calculated_percentiles[percentile] = numpy.percentile(data_list, percentile)<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>" + str(percentile))<EOL><DEDENT><DEDENT>return calculated_stats, calculated_percentiles<EOL>

Calculate statistics for given data.

:param list data_list: List of floats
:param list stats_to_calculate: List of strings with statistics to calculate. Supported stats are defined in constant stats_to_numpy_method_map
:param list percentiles_to_calculate: List of floats that defined which percentiles to calculate.
:return: tuple of dictionaries containing calculated statistics and percentiles


if os.path.exists(filename):<EOL><INDENT>if not os.path.getsize(filename):<EOL><INDENT>logger.warning('<STR_LIT>', filename)<EOL>return False<EOL><DEDENT><DEDENT>else:<EOL><INDENT>logger.warning('<STR_LIT>', filename)<EOL>return False<EOL><DEDENT>return True<EOL>

Check if the specifed file exists and is not empty

:param filename: full path to the file that needs to be checked
:return: Status, Message


time_formats = {<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>'),<EOL>'<STR_LIT>': re.compile(r'<STR_LIT>')<EOL>}<EOL>for time_format in time_formats:<EOL><INDENT>if re.match(time_formats[time_format], timestamp):<EOL><INDENT>return time_format<EOL><DEDENT><DEDENT>return '<STR_LIT>'<EOL>

Given an input timestamp string, determine what format is it likely in.

:param string timestamp: the timestamp string for which we need to determine format
:return: best guess timestamp format


if not timestamp:<EOL><INDENT>return None<EOL><DEDENT>if timestamp == '<STR_LIT>':<EOL><INDENT>timestamp = str(datetime.datetime.now())<EOL><DEDENT>if not ts_format:<EOL><INDENT>ts_format = detect_timestamp_format(timestamp)<EOL><DEDENT>try:<EOL><INDENT>if ts_format == '<STR_LIT>':<EOL><INDENT>logger.error('<STR_LIT>', timestamp)<EOL>return -<NUM_LIT:1><EOL><DEDENT>elif ts_format == '<STR_LIT>':<EOL><INDENT>ts = int(timestamp) * <NUM_LIT:1000><EOL><DEDENT>elif ts_format == '<STR_LIT>':<EOL><INDENT>ts = timestamp<EOL><DEDENT>elif ts_format == '<STR_LIT>':<EOL><INDENT>ts = int(timestamp[:<NUM_LIT:10>]) * <NUM_LIT:1000> + int(timestamp[<NUM_LIT:11>:])<EOL><DEDENT>elif ts_format in ('<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>date_today = str(datetime.date.today())<EOL>dt_obj = datetime.datetime.strptime(date_today + '<STR_LIT:U+0020>' + timestamp, '<STR_LIT>' + ts_format)<EOL>ts = calendar.timegm(dt_obj.utctimetuple()) * <NUM_LIT:1000> + dt_obj.microsecond / <NUM_LIT:1000><EOL><DEDENT>else:<EOL><INDENT>dt_obj = datetime.datetime.strptime(timestamp, ts_format)<EOL>ts = calendar.timegm(dt_obj.utctimetuple()) * <NUM_LIT:1000> + dt_obj.microsecond / <NUM_LIT:1000><EOL><DEDENT><DEDENT>except ValueError:<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>return str(ts)<EOL>

Given a timestamp string, return a time stamp in the epoch ms format. If no date is present in
timestamp then today's date will be added as a prefix before conversion to epoch ms


def set_sla(obj, metric, sub_metric, rules):

if not hasattr(obj, '<STR_LIT>'):<EOL><INDENT>return False<EOL><DEDENT>rules_list = rules.split()<EOL>for rule in rules_list:<EOL><INDENT>if '<STR_LIT:<>' in rule:<EOL><INDENT>stat, threshold = rule.split('<STR_LIT:<>')<EOL>sla = SLA(metric, sub_metric, stat, threshold, '<STR_LIT>')<EOL><DEDENT>elif '<STR_LIT:>>' in rule:<EOL><INDENT>stat, threshold = rule.split('<STR_LIT:>>')<EOL>sla = SLA(metric, sub_metric, stat, threshold, '<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj.logger.error('<STR_LIT>' + rule)<EOL><DEDENT>sla = None<EOL><DEDENT>obj.sla_map[metric][sub_metric][stat] = sla<EOL>if hasattr(obj, '<STR_LIT>'):<EOL><INDENT>obj.sla_list.append(sla)  <EOL><DEDENT><DEDENT>return True<EOL>


if not hasattr(metric, '<STR_LIT>'):<EOL><INDENT>return<EOL><DEDENT>for metric_label in metric.sla_map.keys():<EOL><INDENT>for sub_metric in metric.sla_map[metric_label].keys():<EOL><INDENT>for stat_name in metric.sla_map[metric_label][sub_metric].keys():<EOL><INDENT>sla = metric.sla_map[metric_label][sub_metric][stat_name]<EOL>if stat_name[<NUM_LIT:0>] == '<STR_LIT:p>' and hasattr(metric, '<STR_LIT>'):<EOL><INDENT>if sub_metric in metric.calculated_percentiles.keys():<EOL><INDENT>percentile_num = int(stat_name[<NUM_LIT:1>:])<EOL>if isinstance(percentile_num, float) or isinstance(percentile_num, int):<EOL><INDENT>if percentile_num in metric.calculated_percentiles[sub_metric].keys():<EOL><INDENT>if not sla.check_sla_passed(metric.calculated_percentiles[sub_metric][percentile_num]):<EOL><INDENT>logger.info("<STR_LIT>" + sub_metric)<EOL>metric.status = CONSTANTS.SLA_FAILED<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if sub_metric in metric.calculated_stats.keys() and hasattr(metric, '<STR_LIT>'):<EOL><INDENT>if stat_name in metric.calculated_stats[sub_metric].keys():<EOL><INDENT>if not sla.check_sla_passed(metric.calculated_stats[sub_metric][stat_name]):<EOL><INDENT>logger.info("<STR_LIT>" + sub_metric)<EOL>metric.status = CONSTANTS.SLA_FAILED<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>if len(metric.sla_map.keys()) > <NUM_LIT:0> and hasattr(metric, '<STR_LIT>'):<EOL><INDENT>sla_csv_file = metric.get_sla_csv()<EOL>with open(sla_csv_file, '<STR_LIT:w>') as FH:<EOL><INDENT>for metric_label in metric.sla_map.keys():<EOL><INDENT>for sub_metric in metric.sla_map[metric_label].keys():<EOL><INDENT>for stat, sla in metric.sla_map[metric_label][sub_metric].items():<EOL><INDENT>FH.write('<STR_LIT>' % (sla.get_csv_repr()))<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>

Check if all SLAs pass
:return: 0 (if all SLAs pass) or the number of SLAs failures


def init_logging(logger, log_file, log_level):

with open(log_file, '<STR_LIT:w>'):<EOL><INDENT>pass<EOL><DEDENT>numeric_level = getattr(logging, log_level.upper(), None) if log_level else logging.INFO<EOL>if not isinstance(numeric_level, int):<EOL><INDENT>raise ValueError('<STR_LIT>' % log_level)<EOL><DEDENT>logger.setLevel(logging.DEBUG)<EOL>fh = logging.FileHandler(log_file)<EOL>fh.setLevel(logging.DEBUG)<EOL>ch = logging.StreamHandler()<EOL>ch.setLevel(numeric_level)<EOL>formatter = logging.Formatter('<STR_LIT>')<EOL>fh.setFormatter(formatter)<EOL>ch.setFormatter(formatter)<EOL>logger.addHandler(fh)<EOL>logger.addHandler(ch)<EOL>return CONSTANTS.OK<EOL>

Initialize the naarad logger.
:param: logger: logger object to initialize
:param: log_file: log file name
:param: log_level: log level (debug, info, warn, error)


arg_parser = argparse.ArgumentParser()<EOL>arg_parser.add_argument('<STR_LIT:-c>', '<STR_LIT>', help="<STR_LIT>")<EOL>arg_parser.add_argument('<STR_LIT>', help="<STR_LIT>")<EOL>arg_parser.add_argument('<STR_LIT>', help="<STR_LIT>")<EOL>arg_parser.add_argument('<STR_LIT>', '<STR_LIT>', help="<STR_LIT>")<EOL>arg_parser.add_argument('<STR_LIT>', '<STR_LIT>', help="<STR_LIT>")<EOL>arg_parser.add_argument('<STR_LIT>', '<STR_LIT>', action="<STR_LIT>",<EOL>help="<STR_LIT>"<EOL>"<STR_LIT>")<EOL>arg_parser.add_argument('<STR_LIT>', '<STR_LIT>', help="<STR_LIT>", action="<STR_LIT:store_true>")<EOL>arg_parser.add_argument('<STR_LIT>', '<STR_LIT>', help="<STR_LIT>")<EOL>arg_parser.add_argument('<STR_LIT>', '<STR_LIT>', nargs=<NUM_LIT:2>,<EOL>help="<STR_LIT>"<EOL>"<STR_LIT>", metavar=("<STR_LIT>", "<STR_LIT>"))<EOL>arg_parser.add_argument('<STR_LIT>', '<STR_LIT>',<EOL>help="<STR_LIT>"<EOL>"<STR_LIT>", action="<STR_LIT:store_true>")<EOL>arg_parser.add_argument('<STR_LIT>', '<STR_LIT>', help="<STR_LIT>", action="<STR_LIT:store_true>")<EOL>return arg_parser<EOL>

Initialize list of valid arguments accepted by Naarad CLI
:return: arg_parser: argeparse.ArgumentParser object initialized with naarad CLI parameters


variables_dict = {}<EOL>if args.variables:<EOL><INDENT>for var in args.variables:<EOL><INDENT>words = var.split('<STR_LIT:=>')<EOL>variables_dict[words[<NUM_LIT:0>]] = words[<NUM_LIT:1>]<EOL><DEDENT><DEDENT>return variables_dict<EOL>

Return a dictionary of variables specified at CLI
:param: args: Command Line Arguments namespace


if args.diff:<EOL><INDENT>if not args.output_dir:<EOL><INDENT>logger.error('<STR_LIT>')<EOL>print_usage()<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT><DEDENT>elif not args.output_dir:<EOL><INDENT>print_usage()<EOL>sys.exit(<NUM_LIT:0>)<EOL><DEDENT>

Validate that the necessary argument for normal or diff analysis are specified.
:param: args: Command line arguments namespace


print ("<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL>


metric_list = []<EOL>log_files = os.listdir(input_directory)<EOL>for log_file in log_files:<EOL><INDENT>if log_file in CONSTANTS.SUPPORTED_FILENAME_MAPPING.keys():<EOL><INDENT>metric_list.append(initialize_metric(CONSTANTS.SUPPORTED_FILENAME_MAPPING[log_file], [log_file], None, [], output_directory, CONSTANTS.RESOURCE_PATH,<EOL>CONSTANTS.SUPPORTED_FILENAME_MAPPING[log_file], None, None, {}, None, None, {}))<EOL><DEDENT>else:<EOL><INDENT>logger.warning('<STR_LIT>', log_file)<EOL><DEDENT><DEDENT>return metric_list<EOL>

Auto discover metric types from the files that exist in input_directory and return a list of metrics
:param: input_directory: The location to scan for log files
:param: output_directory: The location for the report


def initialize_metric(section, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, other_options):

metric = None<EOL>metric_type = section.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>if metric_type in metric_classes:<EOL><INDENT>if '<STR_LIT>' in metric_type:<EOL><INDENT>metric = metric_classes['<STR_LIT>'](section, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)<EOL><DEDENT>else:<EOL><INDENT>metric = metric_classes[metric_type](section, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>metric = Metric(section, infile_list, hostname, aggr_metrics, output_directory, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, **other_options)<EOL><DEDENT>return metric<EOL>

Initialize appropriate metric based on type of metric.
:param: section: config section name or auto discovered metric type
:param: infile_list: list of input log files for the metric
:param: hostname: hostname associated with the logs origin
:param: output_directory: report location
:param: resource_path: resource path for report
:param: label: label for config section or auto discovered metric type
:param: ts_start: start time for analysis
:param: ts_end: end time for analysis
:param: rule_strings: list of slas
:param: important_sub_metrics: list of important sub metrics
:param: anomaly_detection_metrics: list of metrics to use for anomaly detection.
:param: other_options: kwargs
:return: metric object


def initialize_aggregate_metric(section, aggr_hosts, aggr_metrics, metrics, outdir_default, resource_path, label, ts_start, ts_end, rule_strings,<EOL>important_sub_metrics, anomaly_detection_metrics, other_options):

metric = None<EOL>metric_type = section.split('<STR_LIT:->')[<NUM_LIT:0>]<EOL>metric = aggregate_metric_classes[metric_type](section, aggr_hosts, aggr_metrics, metrics, outdir_default, resource_path, label, ts_start, ts_end,<EOL>rule_strings, important_sub_metrics, anomaly_detection_metrics, **other_options)<EOL>return metric<EOL>

Initialize aggregate metric
:param: section: config section name
:param: aggr_hosts: list of hostnames to aggregate
:param: aggr_metrics: list of metrics to aggregate
:param: metrics: list of metric objects associated with the current naarad analysis
:param: outdir_default: report location
:param: resource_path: resource path for report
:param: label: label for config section
:param: ts_start: start time for analysis
:param: ts_end: end time for analysis
:param: rule_strings: list of slas
:param: important_sub_metrics: list of important sub metrics
:param: other_options: kwargs
:return: metric object


def handle_single_url(url, outdir, outfile=None):

if not url or type(url) != stror not outdir or type(outdir) != str:<EOL><INDENT>logger.error('<STR_LIT>' % (url, outdir))<EOL>return<EOL><DEDENT>if not naarad.utils.is_valid_url(url):<EOL><INDENT>logger.error("<STR_LIT>" % url)<EOL>return<EOL><DEDENT>if not outfile:<EOL><INDENT>segs = url.split('<STR_LIT:/>')<EOL>outfile = segs[-<NUM_LIT:1>]<EOL>outfile = urllib2.quote(outfile)<EOL><DEDENT>output_file = os.path.join(outdir, outfile)<EOL>if os.path.exists(output_file):<EOL><INDENT>logger.warn("<STR_LIT>" % outfile)<EOL><DEDENT>with open(output_file, "<STR_LIT:w>") as fh:<EOL><INDENT>try:<EOL><INDENT>response = urllib2.urlopen(url)<EOL>fh.write(response.read())<EOL><DEDENT>except urllib2.HTTPError:<EOL><INDENT>logger.error("<STR_LIT>" % url)<EOL>return<EOL><DEDENT>except urllib2.URLError:<EOL><INDENT>logger.error("<STR_LIT>" % url)<EOL>return<EOL><DEDENT><DEDENT>return output_file<EOL>

Base function which takes a single url, download it to outdir/outfile
:param str url: a full/absolute url, e.g. http://www.cnn.com/log.zip
:param str outdir: the absolute local directory. e.g. /home/user1/tmp/
:param str outfile: (optional) filename stored in local directory. If outfile is not given, extract the filename from url
:return: the local full path name of downloaded url


try:<EOL><INDENT>response = urllib2.urlopen(url)<EOL>response_content = response.read()<EOL>return response_content<EOL><DEDENT>except (urllib2.URLError, urllib2.HTTPError) as e:<EOL><INDENT>logger.error('<STR_LIT>', url)<EOL>return False<EOL><DEDENT>

Read response of specified url into memory and return to caller. No persistence to disk.
:return: response content if accessing the URL succeeds, False otherwise


if not url or type(url) != str or not naarad.utils.is_valid_url(url):<EOL><INDENT>logger.error("<STR_LIT>")<EOL>return<EOL><DEDENT>base_index = url.find('<STR_LIT:/>', len("<STR_LIT>"))   <EOL>base_url = url[:base_index]      <EOL>urls = []<EOL>try:<EOL><INDENT>response = urllib2.urlopen(url)<EOL>hp = HTMLLinkExtractor()<EOL>hp.feed(response.read())<EOL>urls = hp.links<EOL>hp.close()<EOL><DEDENT>except urllib2.HTTPError:<EOL><INDENT>logger.error("<STR_LIT>" % url)<EOL>return urls<EOL><DEDENT>for i in range(len(urls)):<EOL><INDENT>if not urls[i].startswith("<STR_LIT>") and not urls[i].startswith("<STR_LIT>"):    <EOL><INDENT>urls[i] = base_url + urls[i]<EOL><DEDENT><DEDENT>return urls<EOL>

get a list of urls from a seeding url, return a list of urls

:param str url: a full/absolute url, e.g. http://www.cnn.com/logs/
:return: a list of full/absolute urls.


def download_url_single(inputs, outdir, outfile=None):

if not inputs or type(inputs) != str or not outdir or type(outdir) != str:<EOL><INDENT>logging.error("<STR_LIT>")<EOL>return<EOL><DEDENT>else:<EOL><INDENT>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT><DEDENT>output_file = handle_single_url(inputs, outdir, outfile)<EOL>return output_file<EOL>

Downloads a http(s) url to a local file
:param str inputs:  the absolute url
:param str outdir: Required. the local directory to put the downloadedfiles.
:param str outfile: // Optional. If this is given, the downloaded url will be renated to outfile;
  If this is not given, then the local file will be the original one, as given in url.
:return: the local full path name of downloaded url


def download_url_regex(inputs, outdir, regex="<STR_LIT>"):

if not inputs or type(inputs) != stror not outdir or type(outdir) != str:<EOL><INDENT>logging.error("<STR_LIT>")<EOL>return<EOL><DEDENT>else:<EOL><INDENT>if not os.path.exists(outdir):<EOL><INDENT>os.makedirs(outdir)<EOL><DEDENT><DEDENT>output_files = []<EOL>files = get_urls_from_seed(inputs)<EOL>for f in files:<EOL><INDENT>if re.compile(regex).match(f):<EOL><INDENT>output_file = handle_single_url(f, outdir)<EOL>output_files.append(output_file)<EOL><DEDENT><DEDENT>return output_files<EOL>

Downloads http(s) urls to a local files
:param str inputs: Required, the seed url
:param str outdir: Required. the local directory to put the downloadedfiles.
:param str regex: Optional, a regex string. If not given, then all urls will be valid
:return: A list of local full path names (downloaded from inputs)


for url in url_list:<EOL><INDENT>download_url_single(url, outdir)<EOL><DEDENT>

Downloads list of http(s) urls to local files
:param list url_list: list of URLs to download
:param str outdir: Required. the local directory to put the downloadedfiles.
  If this is not given, then the local file will be the original one, as given in url.
:return None


cmd_args = shlex.split(self.run_cmd)<EOL>logger.info('<STR_LIT>', self.run_rank)<EOL>logger.info('<STR_LIT>' + str(cmd_args))<EOL>self.ts_start = time.strftime("<STR_LIT>")<EOL>try:<EOL><INDENT>self.process = subprocess.Popen(cmd_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=<NUM_LIT:1>)<EOL>if self.kill_after_seconds:<EOL><INDENT>self.timer = Timer(self.kill_after_seconds, self.kill)<EOL>self.timer.start()<EOL><DEDENT>for line in iter(self.process.stdout.readline, b'<STR_LIT>'):<EOL><INDENT>logger.info(line.strip())<EOL><DEDENT>self.process.communicate()<EOL><DEDENT>except KeyboardInterrupt:<EOL><INDENT>logger.warning('<STR_LIT>')<EOL>self.kill()<EOL><DEDENT>if self.timer:<EOL><INDENT>self.timer.cancel()<EOL><DEDENT>self.ts_end = time.strftime("<STR_LIT>")<EOL>logger.info('<STR_LIT>')<EOL>logger.info('<STR_LIT>' + self.ts_start + '<STR_LIT>' + self.ts_end)<EOL>

Run the command, infer time period to be used in metric analysis phase.
:return: None


try:<EOL><INDENT>logger.info('<STR_LIT>')<EOL>self.process.terminate()<EOL>time_waited_seconds = <NUM_LIT:0><EOL>while self.process.poll() is None and time_waited_seconds < CONSTANTS.SECONDS_TO_KILL_AFTER_SIGTERM:<EOL><INDENT>time.sleep(<NUM_LIT:0.5>)<EOL>time_waited_seconds += <NUM_LIT:0.5><EOL><DEDENT>if self.process.poll() is None:<EOL><INDENT>self.process.kill()<EOL>logger.warning('<STR_LIT>', CONSTANTS.SECONDS_TO_KILL_AFTER_SIGTERM)<EOL><DEDENT><DEDENT>except OSError as e:<EOL><INDENT>logger.error('<STR_LIT>', e)<EOL><DEDENT>

If run_step needs to be killed, this method will be called
:return: None


def __init__(self, run_type, run_cmd, call_type, run_order, run_rank, should_wait=True, kill_after_seconds=None):

self.run_type = run_type<EOL>self.run_cmd = run_cmd<EOL>self.call_type = call_type<EOL>self.run_order = run_order<EOL>self.run_rank = run_rank<EOL>self.should_wait = should_wait<EOL>self.kill_after_seconds = kill_after_seconds<EOL>self.timer = None<EOL>

Init method
:param run_type: Type of run_step: "workload" only for now
:param run_cmd: Details of command to be run. It could be a command or API call
:param call_type: Kind of call -- local or remote
:param run_order: When to run this w.r.t analysis. One of ('pre', 'in', 'post')
:param run_rank: In what order to run this
:param should_wait: Boolean whether naarad should wait for the run command to finish or not before moving on
:param kill_after_seconds: Seconds for which the command should be run before being killed
:return: None


return pkg_resources.resource_filename('<STR_LIT>', None)<EOL>

Return the location of resources for report


config = ConfigParser()<EOL>config.read(filename)<EOL>working_dict = _prepare_working_dict(config, section_option_dict)<EOL>tmp_dict = {}<EOL>for section, options in working_dict.iteritems():<EOL><INDENT>tmp_dict[section] = {}<EOL>for option in options:<EOL><INDENT>tmp_dict[section][option] = config.get(section, option)<EOL><DEDENT><DEDENT>return Bunch(tmp_dict)<EOL>

This function returns a Bunch object from the stated config file.

!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
NOTE:
    The values are not evaluated by default.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

filename:
    The desired config file to read.
    The config file must be written in a syntax readable to the
    ConfigParser module -> INI syntax

    [sectionA]
    optionA1 = ...
    optionA2 = ...

section_option_dict:
    A dictionary that contains keys, which are associated to the sections
    in the config file, and values, which are a list of the desired
    options.
    If empty, everything will be loaded.
    If the lists are empty, everything from the sections will be loaded.

Example:
    dict = {'sectionA': ['optionA1', 'optionA2', ...],
            'sectionB': ['optionB1', 'optionB2', ...]}

    config = get_config('config.cfg', dict)
    config.sectionA.optionA1

Other:
    Bunch can be found in configparser.py


HTMLParser.__init__(self)<EOL>self.reset()<EOL>self.fed = []<EOL>


self.fed.append(d)<EOL>

Return representation of pure text data.


if tag in self.mathml_elements:<EOL><INDENT>final_attr = "<STR_LIT>"<EOL>for key, value in attrs:<EOL><INDENT>final_attr += '<STR_LIT>'.format(key, value)<EOL><DEDENT>self.fed.append("<STR_LIT>".format(tag, final_attr))<EOL><DEDENT>

Return representation of html start tag and attributes.


if tag in self.mathml_elements:<EOL><INDENT>self.fed.append("<STR_LIT>".format(tag))<EOL><DEDENT>


self.fed.append('<STR_LIT>' % name)<EOL>


Return representation of numeric entities.


return '<STR_LIT>'.join(self.fed)<EOL>


s = cls()<EOL>s.feed(html)<EOL>unescaped_data = s.unescape(s.get_data())<EOL>return escape_for_xml(unescaped_data, tags_to_keep=s.mathml_elements)<EOL>

Return stripped HTML, keeping only MathML.


page = urllib2.urlopen(url)<EOL>pages = [BeautifulSoup(page)]<EOL>numpag = pages[<NUM_LIT:0>].body.findAll('<STR_LIT>', attrs={'<STR_LIT:class>': '<STR_LIT>'})<EOL>if len(numpag) > <NUM_LIT:0>:<EOL><INDENT>if re.search('<STR_LIT>', numpag[<NUM_LIT:0>].string):<EOL><INDENT>for i in range(int(numpag[<NUM_LIT:0>].string)-<NUM_LIT:1>):<EOL><INDENT>page = urllib2.urlopen('<STR_LIT>' % (url, i+<NUM_LIT:2>))<EOL>pages.append(BeautifulSoup(page))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>print("<STR_LIT>" % (numpag[<NUM_LIT:0>].string))<EOL><DEDENT><DEDENT>impl = getDOMImplementation()<EOL>doc = impl.createDocument(None, "<STR_LIT>", None)<EOL>links = []<EOL>for page in pages:<EOL><INDENT>links += page.body.findAll('<STR_LIT:p>', attrs={'<STR_LIT:class>': '<STR_LIT:title>'})<EOL>links += page.body.findAll('<STR_LIT>', attrs={'<STR_LIT:class>': '<STR_LIT:title>'})<EOL><DEDENT>for link in links:<EOL><INDENT>record = self._get_record(link)<EOL>doc.firstChild.appendChild(record)<EOL><DEDENT>return doc.toprettyxml()<EOL>

Returns the records listed in the webpage given as
parameter as a xml String.

@param url: the url of the Journal, Book, Protocol or Reference work


def create_field(subfields=None, ind1='<STR_LIT:U+0020>', ind2='<STR_LIT:U+0020>', controlfield_value='<STR_LIT>',<EOL>global_position=-<NUM_LIT:1>):

if subfields is None:<EOL><INDENT>subfields = []<EOL><DEDENT>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>field = (subfields, ind1, ind2, controlfield_value, global_position)<EOL>_check_field_validity(field)<EOL>return field<EOL>

Return a field created with the provided elements.

Global position is set arbitrary to -1.


<EOL>regex = re.compile('<STR_LIT>', re.DOTALL)<EOL>record_xmls = regex.findall(marcxml)<EOL>return [create_record(record_xml, verbose=verbose, correct=correct,<EOL>parser=parser, keep_singletons=keep_singletons)<EOL>for record_xml in record_xmls]<EOL>

Create a list of records from the marcxml description.

:returns: a list of objects initiated by the function create_record().
          Please see that function's docstring.


if marcxml is None:<EOL><INDENT>return {}<EOL><DEDENT>try:<EOL><INDENT>rec = _create_record_lxml(marcxml, verbose, correct,<EOL>keep_singletons=keep_singletons)<EOL><DEDENT>except InvenioBibRecordParserError as ex1:<EOL><INDENT>return (None, <NUM_LIT:0>, str(ex1))<EOL><DEDENT>if sort_fields_by_indicators:<EOL><INDENT>_record_sort_by_indicators(rec)<EOL><DEDENT>errs = []<EOL>if correct:<EOL><INDENT>errs = _correct_record(rec)<EOL><DEDENT>return (rec, int(not errs), errs)<EOL>


def filter_field_instances(field_instances, filter_subcode, filter_value,<EOL>filter_mode='<STR_LIT:e>'):

matched = []<EOL>if filter_mode == '<STR_LIT:e>':<EOL><INDENT>to_match = (filter_subcode, filter_value)<EOL>for instance in field_instances:<EOL><INDENT>if to_match in instance[<NUM_LIT:0>]:<EOL><INDENT>matched.append(instance)<EOL><DEDENT><DEDENT><DEDENT>elif filter_mode == '<STR_LIT:s>':<EOL><INDENT>for instance in field_instances:<EOL><INDENT>for subfield in instance[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == filter_subcode andsubfield[<NUM_LIT:1>].find(filter_value) > -<NUM_LIT:1>:<EOL><INDENT>matched.append(instance)<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif filter_mode == '<STR_LIT:r>':<EOL><INDENT>reg_exp = re.compile(filter_value)<EOL>for instance in field_instances:<EOL><INDENT>for subfield in instance[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == filter_subcode andreg_exp.match(subfield[<NUM_LIT:1>]) is not None:<EOL><INDENT>matched.append(instance)<EOL>break<EOL><DEDENT><DEDENT><DEDENT><DEDENT>return matched<EOL>

Filter the given field.

    Filters given field and returns only that field instances that contain
    filter_subcode with given filter_value. As an input for search function
    accepts output from record_get_field_instances function. Function can be
    run in three modes:

    - 'e' - looking for exact match in subfield value
    - 's' - looking for substring in subfield value
    - 'r' - looking for regular expression in subfield value

    Example:

    record_filter_field(record_get_field_instances(rec, '999', '%', '%'),
                        'y', '2001')

    In this case filter_subcode is 'y' and filter_value is '2001'.

    :param field_instances: output from record_get_field_instances
    :param filter_subcode: name of the subfield
    :type filter_subcode: string
    :param filter_value: value of the subfield
    :type filter_value: string
    :param filter_mode: 'e','s' or 'r'


out = {}<EOL>position = <NUM_LIT:0><EOL>tags = sorted(record.keys())<EOL>for tag in tags:<EOL><INDENT>fields = record[tag]<EOL>out[tag] = []<EOL>current_fields = set()<EOL>for full_field in fields:<EOL><INDENT>field = (tuple(full_field[<NUM_LIT:0>]),) + full_field[<NUM_LIT:1>:<NUM_LIT:4>]<EOL>if field not in current_fields:<EOL><INDENT>current_fields.add(field)<EOL>position += <NUM_LIT:1><EOL>out[tag].append(full_field[:<NUM_LIT:4>] + (position,))<EOL><DEDENT><DEDENT><DEDENT>return out<EOL>

Return a record where all the duplicate fields have been removed.

Fields are considered identical considering also the order of their
subfields.


rec1_keys = set(rec1.keys())<EOL>rec2_keys = set(rec2.keys())<EOL>if skip_005:<EOL><INDENT>rec1_keys.discard("<STR_LIT>")<EOL>rec2_keys.discard("<STR_LIT>")<EOL><DEDENT>if rec1_keys != rec2_keys:<EOL><INDENT>return False<EOL><DEDENT>for key in rec1_keys:<EOL><INDENT>if ignore_duplicate_controlfields and key.startswith('<STR_LIT>'):<EOL><INDENT>if set(field[<NUM_LIT:3>] for field in rec1[key]) !=set(field[<NUM_LIT:3>] for field in rec2[key]):<EOL><INDENT>return False<EOL><DEDENT>continue<EOL><DEDENT>rec1_fields = rec1[key]<EOL>rec2_fields = rec2[key]<EOL>if len(rec1_fields) != len(rec2_fields):<EOL><INDENT>return False<EOL><DEDENT>if ignore_field_order:<EOL><INDENT>rec1_fields = sorted(<EOL>rec1_fields,<EOL>key=lambda elem: (elem[<NUM_LIT:1>], elem[<NUM_LIT:2>], elem[<NUM_LIT:3>], elem[<NUM_LIT:0>]))<EOL>rec2_fields = sorted(<EOL>rec2_fields,<EOL>key=lambda elem: (elem[<NUM_LIT:1>], elem[<NUM_LIT:2>], elem[<NUM_LIT:3>], elem[<NUM_LIT:0>]))<EOL><DEDENT>else:<EOL><INDENT>rec1_fields = sorted(<EOL>rec1_fields,<EOL>key=lambda elem: (elem[<NUM_LIT:1>], elem[<NUM_LIT:2>], elem[<NUM_LIT:4>], elem[<NUM_LIT:3>], elem[<NUM_LIT:0>]))<EOL>rec2_fields = sorted(<EOL>rec2_fields,<EOL>key=lambda elem: (elem[<NUM_LIT:1>], elem[<NUM_LIT:2>], elem[<NUM_LIT:4>], elem[<NUM_LIT:3>], elem[<NUM_LIT:0>]))<EOL><DEDENT>for field1, field2 in zip(rec1_fields, rec2_fields):<EOL><INDENT>if ignore_duplicate_subfields:<EOL><INDENT>if field1[<NUM_LIT:1>:<NUM_LIT:4>] != field2[<NUM_LIT:1>:<NUM_LIT:4>] orset(field1[<NUM_LIT:0>]) != set(field2[<NUM_LIT:0>]):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>elif ignore_subfield_order:<EOL><INDENT>if field1[<NUM_LIT:1>:<NUM_LIT:4>] != field2[<NUM_LIT:1>:<NUM_LIT:4>] orsorted(field1[<NUM_LIT:0>]) != sorted(field2[<NUM_LIT:0>]):<EOL><INDENT>return False<EOL><DEDENT><DEDENT>elif field1[:<NUM_LIT:4>] != field2[:<NUM_LIT:4>]:<EOL><INDENT>return False<EOL><DEDENT><DEDENT><DEDENT>return True<EOL>

Return True if rec1 is identical to rec2.

It does so regardless of a difference in the 005 tag (i.e. the timestamp).


def record_get_field_instances(rec, tag="<STR_LIT>", ind1="<STR_LIT:U+0020>", ind2="<STR_LIT:U+0020>"):

if not rec:<EOL><INDENT>return []<EOL><DEDENT>if not tag:<EOL><INDENT>return list(rec.items())<EOL><DEDENT>else:<EOL><INDENT>out = []<EOL>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>if '<STR_LIT:%>' in tag:<EOL><INDENT>for field_tag in rec:<EOL><INDENT>if _tag_matches_pattern(field_tag, tag):<EOL><INDENT>for possible_field_instance in rec[field_tag]:<EOL><INDENT>if (ind1 in ('<STR_LIT:%>', possible_field_instance[<NUM_LIT:1>]) and<EOL>ind2 in ('<STR_LIT:%>', possible_field_instance[<NUM_LIT:2>])):<EOL><INDENT>out.append(possible_field_instance)<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for possible_field_instance in rec.get(tag, []):<EOL><INDENT>if (ind1 in ('<STR_LIT:%>', possible_field_instance[<NUM_LIT:1>]) and<EOL>ind2 in ('<STR_LIT:%>', possible_field_instance[<NUM_LIT:2>])):<EOL><INDENT>out.append(possible_field_instance)<EOL><DEDENT><DEDENT><DEDENT>return out<EOL><DEDENT>

Return the list of field instances for the specified tag and indications.

Return empty list if not found.
If tag is empty string, returns all fields

Parameters (tag, ind1, ind2) can contain wildcard %.

:param rec: a record structure as returned by create_record()
:param tag: a 3 characters long string
:param ind1: a 1 character long string
:param ind2: a 1 character long string
:param code: a 1 character long string
:return: a list of field tuples (Subfields, ind1, ind2, value,
         field_position_global) where subfields is list of (code, value)


def record_add_field(rec, tag, ind1='<STR_LIT:U+0020>', ind2='<STR_LIT:U+0020>', controlfield_value='<STR_LIT>',<EOL>subfields=None, field_position_global=None,<EOL>field_position_local=None):

error = _validate_record_field_positions_global(rec)<EOL>if error:<EOL><INDENT>pass<EOL><DEDENT>if subfields is None:<EOL><INDENT>subfields = []<EOL><DEDENT>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>if controlfield_value and (ind1 != '<STR_LIT:U+0020>' or ind2 != '<STR_LIT:U+0020>' or subfields):<EOL><INDENT>return -<NUM_LIT:1><EOL><DEDENT>tag_field_positions_global = {}.fromkeys([field[<NUM_LIT:4>]<EOL>for field in rec.get(tag, [])])<EOL>all_field_positions_global = {}.fromkeys([field[<NUM_LIT:4>]<EOL>for fields in list(rec.values())<EOL>for field in fields])<EOL>if field_position_global is None and field_position_local is None:<EOL><INDENT>if tag in rec:<EOL><INDENT>try:<EOL><INDENT>field_position_global = max([field[<NUM_LIT:4>] for field in rec[tag]])+ <NUM_LIT:1><EOL><DEDENT>except IndexError:<EOL><INDENT>if tag_field_positions_global:<EOL><INDENT>field_position_global = max(tag_field_positions_global) + <NUM_LIT:1><EOL><DEDENT>elif all_field_positions_global:<EOL><INDENT>field_position_global = max(all_field_positions_global) + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>field_position_global = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if tag in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>'):<EOL><INDENT>if tag_field_positions_global:<EOL><INDENT>field_position_global = max(tag_field_positions_global) + <NUM_LIT:1><EOL><DEDENT>elif all_field_positions_global:<EOL><INDENT>field_position_global = max(all_field_positions_global) + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>field_position_global = <NUM_LIT:1><EOL><DEDENT><DEDENT>else:<EOL><INDENT>immediate_lower_tag = '<STR_LIT>'<EOL>for rec_tag in rec:<EOL><INDENT>if (tag not in ('<STR_LIT>', '<STR_LIT>', '<STR_LIT>', '<STR_LIT>') and<EOL>rec[rec_tag] and<EOL>immediate_lower_tag < rec_tag < tag):<EOL><INDENT>immediate_lower_tag = rec_tag<EOL><DEDENT><DEDENT>if immediate_lower_tag == '<STR_LIT>':<EOL><INDENT>field_position_global = <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>field_position_global = rec[immediate_lower_tag][-<NUM_LIT:1>][<NUM_LIT:4>] + <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>field_position_local = len(rec.get(tag, []))<EOL>_shift_field_positions_global(rec, field_position_global, <NUM_LIT:1>)<EOL><DEDENT>elif field_position_local is not None:<EOL><INDENT>if tag in rec:<EOL><INDENT>if field_position_local >= len(rec[tag]):<EOL><INDENT>field_position_global = rec[tag][-<NUM_LIT:1>][<NUM_LIT:4>] + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>field_position_global = rec[tag][field_position_local][<NUM_LIT:4>]<EOL><DEDENT>_shift_field_positions_global(rec, field_position_global, <NUM_LIT:1>)<EOL><DEDENT>else:<EOL><INDENT>if all_field_positions_global:<EOL><INDENT>field_position_global = max(all_field_positions_global) + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>field_position_global = <NUM_LIT:1><EOL><DEDENT><DEDENT><DEDENT>elif field_position_global is not None:<EOL><INDENT>if tag not in rec:<EOL><INDENT>if all_field_positions_global:<EOL><INDENT>field_position_global = max(all_field_positions_global) + <NUM_LIT:1><EOL><DEDENT>else:<EOL><INDENT>field_position_global = <NUM_LIT:1><EOL><DEDENT>field_position_local = <NUM_LIT:0><EOL><DEDENT>elif field_position_global < min(tag_field_positions_global):<EOL><INDENT>field_position_global = min(tag_field_positions_global)<EOL>_shift_field_positions_global(rec, min(tag_field_positions_global),<EOL><NUM_LIT:1>)<EOL>field_position_local = <NUM_LIT:0><EOL><DEDENT>elif field_position_global > max(tag_field_positions_global):<EOL><INDENT>field_position_global = max(tag_field_positions_global) + <NUM_LIT:1><EOL>_shift_field_positions_global(rec,<EOL>max(tag_field_positions_global) + <NUM_LIT:1>,<EOL><NUM_LIT:1>)<EOL>field_position_local = len(rec.get(tag, []))<EOL><DEDENT>else:<EOL><INDENT>if field_position_global in tag_field_positions_global:<EOL><INDENT>_shift_field_positions_global(rec, field_position_global, <NUM_LIT:1>)<EOL><DEDENT>field_position_local = <NUM_LIT:0><EOL>for position, field in enumerate(rec[tag]):<EOL><INDENT>if field[<NUM_LIT:4>] == field_position_global + <NUM_LIT:1>:<EOL><INDENT>field_position_local = position<EOL><DEDENT><DEDENT><DEDENT><DEDENT>newfield = (subfields, ind1, ind2, str(controlfield_value),<EOL>field_position_global)<EOL>rec.setdefault(tag, []).insert(field_position_local, newfield)<EOL>return field_position_global<EOL>

Add a new field into the record.

If field_position_global or field_position_local is specified then
this method will insert the new field at the desired position.
Otherwise a global field position will be computed in order to
insert the field at the best position (first we try to keep the
order of the tags and then we insert the field at the end of the
fields with the same tag).

If both field_position_global and field_position_local are present,
then field_position_local takes precedence.

:param rec: the record data structure
:param tag: the tag of the field to be added
:param ind1: the first indicator
:param ind2: the second indicator
:param controlfield_value: the value of the controlfield
:param subfields: the subfields (a list of tuples (code, value))
:param field_position_global: the global field position (record wise)
:param field_position_local: the local field position (tag wise)
:return: the global field position of the newly inserted field or -1 if the
         operation failed


return tag in rec<EOL>

Check if the tag exists in the record.

:param rec: the record data structure
:param the: field
:return: a boolean


def record_delete_field(rec, tag, ind1='<STR_LIT:U+0020>', ind2='<STR_LIT:U+0020>',<EOL>field_position_global=None, field_position_local=None):

error = _validate_record_field_positions_global(rec)<EOL>if error:<EOL><INDENT>pass<EOL><DEDENT>if tag not in rec:<EOL><INDENT>return False<EOL><DEDENT>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>deleted = []<EOL>newfields = []<EOL>if field_position_global is None and field_position_local is None:<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>if field[<NUM_LIT:1>] != ind1 or field[<NUM_LIT:2>] != ind2:<EOL><INDENT>newfields.append(field)<EOL><DEDENT>else:<EOL><INDENT>deleted.append(field)<EOL><DEDENT><DEDENT>rec[tag] = newfields<EOL><DEDENT>elif field_position_global is not None:<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>if (field[<NUM_LIT:1>] != ind1 and field[<NUM_LIT:2>] != ind2 or<EOL>field[<NUM_LIT:4>] != field_position_global):<EOL><INDENT>newfields.append(field)<EOL><DEDENT>else:<EOL><INDENT>deleted.append(field)<EOL><DEDENT><DEDENT>rec[tag] = newfields<EOL><DEDENT>elif field_position_local is not None:<EOL><INDENT>try:<EOL><INDENT>del rec[tag][field_position_local]<EOL><DEDENT>except IndexError:<EOL><INDENT>return []<EOL><DEDENT><DEDENT>if not rec[tag]:<EOL><INDENT>del rec[tag]<EOL><DEDENT>return deleted<EOL>

Delete the field with the given position.

If global field position is specified, deletes the field with the
corresponding global field position.
If field_position_local is specified, deletes the field with the
corresponding local field position and tag.
Else deletes all the fields matching tag and optionally ind1 and
ind2.

If both field_position_global and field_position_local are present,
then field_position_local takes precedence.

:param rec: the record data structure
:param tag: the tag of the field to be deleted
:param ind1: the first indicator of the field to be deleted
:param ind2: the second indicator of the field to be deleted
:param field_position_global: the global field position (record wise)
:param field_position_local: the local field position (tag wise)
:return: the list of deleted fields


def record_delete_fields(rec, tag, field_positions_local=None):

if tag not in rec:<EOL><INDENT>return []<EOL><DEDENT>new_fields, deleted_fields = [], []<EOL>for position, field in enumerate(rec.get(tag, [])):<EOL><INDENT>if field_positions_local is None or position in field_positions_local:<EOL><INDENT>deleted_fields.append(field)<EOL><DEDENT>else:<EOL><INDENT>new_fields.append(field)<EOL><DEDENT><DEDENT>if new_fields:<EOL><INDENT>rec[tag] = new_fields<EOL><DEDENT>else:<EOL><INDENT>del rec[tag]<EOL><DEDENT>return deleted_fields<EOL>

Delete all/some fields defined with MARC tag 'tag' from record 'rec'.

:param rec: a record structure.
:type rec: tuple
:param tag: three letter field.
:type tag: string
:param field_position_local: if set, it is the list of local positions
    within all the fields with the specified tag, that should be deleted.
    If not set all the fields with the specified tag will be deleted.
:type field_position_local: sequence
:return: the list of deleted fields.
:rtype: list
:note: the record is modified in place.


def record_add_fields(rec, tag, fields, field_position_local=None,<EOL>field_position_global=None):

if field_position_local is None and field_position_global is None:<EOL><INDENT>for field in fields:<EOL><INDENT>record_add_field(<EOL>rec, tag, ind1=field[<NUM_LIT:1>],<EOL>ind2=field[<NUM_LIT:2>], subfields=field[<NUM_LIT:0>],<EOL>controlfield_value=field[<NUM_LIT:3>])<EOL><DEDENT><DEDENT>else:<EOL><INDENT>fields.reverse()<EOL>for field in fields:<EOL><INDENT>record_add_field(<EOL>rec, tag, ind1=field[<NUM_LIT:1>], ind2=field[<NUM_LIT:2>],<EOL>subfields=field[<NUM_LIT:0>], controlfield_value=field[<NUM_LIT:3>],<EOL>field_position_local=field_position_local,<EOL>field_position_global=field_position_global)<EOL><DEDENT><DEDENT>return field_position_local<EOL>

Add the fields into the record at the required position.

The position is specified by the tag and the field_position_local in the
list of fields.

:param rec: a record structure
:param tag: the tag of the fields to be moved
:param field_position_local: the field_position_local to which the field
                             will be inserted. If not specified, appends
                             the fields to the tag.
:param a: list of fields to be added
:return: -1 if the operation failed, or the field_position_local if it was
         successful


def record_move_fields(rec, tag, field_positions_local,<EOL>field_position_local=None):

fields = record_delete_fields(<EOL>rec, tag,<EOL>field_positions_local=field_positions_local)<EOL>return record_add_fields(<EOL>rec, tag, fields,<EOL>field_position_local=field_position_local)<EOL>

Move some fields to the position specified by 'field_position_local'.

:param rec: a record structure as returned by create_record()
:param tag: the tag of the fields to be moved
:param field_positions_local: the positions of the fields to move
:param field_position_local: insert the field before that
                             field_position_local. If unspecified, appends
                             the fields :return: the field_position_local
                             is the operation was successful


def record_delete_subfield(rec, tag, subfield_code, ind1='<STR_LIT:U+0020>', ind2='<STR_LIT:U+0020>'):

ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>for field in rec.get(tag, []):<EOL><INDENT>if field[<NUM_LIT:1>] == ind1 and field[<NUM_LIT:2>] == ind2:<EOL><INDENT>field[<NUM_LIT:0>][:] = [subfield for subfield in field[<NUM_LIT:0>]<EOL>if subfield_code != subfield[<NUM_LIT:0>]]<EOL><DEDENT><DEDENT>

Delete all subfields with subfield_code in the record.


def record_get_field(rec, tag, field_position_global=None,<EOL>field_position_local=None):

if field_position_global is None and field_position_local is None:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>elif field_position_global is not None andfield_position_local is not None:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>elif field_position_global:<EOL><INDENT>if tag not in rec:<EOL><INDENT>raise InvenioBibRecordFieldError("<STR_LIT>" % tag)<EOL><DEDENT>for field in rec[tag]:<EOL><INDENT>if field[<NUM_LIT:4>] == field_position_global:<EOL><INDENT>return field<EOL><DEDENT><DEDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (tag, field_position_global))<EOL><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>return rec[tag][field_position_local]<EOL><DEDENT>except KeyError:<EOL><INDENT>raise InvenioBibRecordFieldError("<STR_LIT>" % tag)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (tag, field_position_local))<EOL><DEDENT><DEDENT>

Return the the matching field.

One has to enter either a global field position or a local field position.

:return: a list of subfield tuples (subfield code, value).
:rtype: list


def record_replace_field(rec, tag, new_field, field_position_global=None,<EOL>field_position_local=None):

if field_position_global is None and field_position_local is None:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>elif field_position_global is not None andfield_position_local is not None:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>")<EOL><DEDENT>elif field_position_global:<EOL><INDENT>if tag not in rec:<EOL><INDENT>raise InvenioBibRecordFieldError("<STR_LIT>" % tag)<EOL><DEDENT>replaced = False<EOL>for position, field in enumerate(rec[tag]):<EOL><INDENT>if field[<NUM_LIT:4>] == field_position_global:<EOL><INDENT>rec[tag][position] = new_field<EOL>replaced = True<EOL><DEDENT><DEDENT>if not replaced:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>(tag, field_position_global))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>try:<EOL><INDENT>rec[tag][field_position_local] = new_field<EOL><DEDENT>except KeyError:<EOL><INDENT>raise InvenioBibRecordFieldError("<STR_LIT>" % tag)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" % (tag, field_position_local))<EOL><DEDENT><DEDENT>


def record_get_subfields(rec, tag, field_position_global=None,<EOL>field_position_local=None):

field = record_get_field(<EOL>rec, tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>return field[<NUM_LIT:0>]<EOL>

Return the subfield of the matching field.

One has to enter either a global field position or a local field position.

:return: a list of subfield tuples (subfield code, value).
:rtype:  list


subfields = record_get_subfields(<EOL>rec, tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>try:<EOL><INDENT>del subfields[subfield_position]<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>"<EOL>"<STR_LIT>" %<EOL>{"<STR_LIT>": subfield_position,<EOL>"<STR_LIT>": str(field_position_local),<EOL>"<STR_LIT>": str(field_position_global),<EOL>"<STR_LIT>": tag})<EOL><DEDENT>if not subfields:<EOL><INDENT>if field_position_global is not None:<EOL><INDENT>for position, field in enumerate(rec[tag]):<EOL><INDENT>if field[<NUM_LIT:4>] == field_position_global:<EOL><INDENT>del rec[tag][position]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>del rec[tag][field_position_local]<EOL><DEDENT>if not rec[tag]:<EOL><INDENT>del rec[tag]<EOL><DEDENT><DEDENT>

Delete subfield from position specified.

Specify the subfield by tag, field number and subfield position.


def record_add_subfield_into(rec, tag, subfield_code, value,<EOL>subfield_position=None,<EOL>field_position_global=None,<EOL>field_position_local=None):

subfields = record_get_subfields(<EOL>rec, tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>if subfield_position is None:<EOL><INDENT>subfields.append((subfield_code, value))<EOL><DEDENT>else:<EOL><INDENT>subfields.insert(subfield_position, (subfield_code, value))<EOL><DEDENT>

Add subfield into specified position.

    Specify the subfield by tag, field number and optionally by subfield
    position.


field = record_get_field(<EOL>rec, tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>new_field = (field[<NUM_LIT:0>], field[<NUM_LIT:1>], field[<NUM_LIT:2>], controlfield_value, field[<NUM_LIT:4>])<EOL>record_replace_field(<EOL>rec, tag, new_field,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>

Modify controlfield at position specified by tag and field number.


def record_modify_subfield(rec, tag, subfield_code, value, subfield_position,<EOL>field_position_global=None,<EOL>field_position_local=None):

subfields = record_get_subfields(<EOL>rec, tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>try:<EOL><INDENT>subfields[subfield_position] = (subfield_code, value)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>" % subfield_position)<EOL><DEDENT>

Modify subfield at specified position.

    Specify the subfield by tag, field number and subfield position.


def record_move_subfield(rec, tag, subfield_position, new_subfield_position,<EOL>field_position_global=None,<EOL>field_position_local=None):

subfields = record_get_subfields(<EOL>rec,<EOL>tag,<EOL>field_position_global=field_position_global,<EOL>field_position_local=field_position_local)<EOL>try:<EOL><INDENT>subfield = subfields.pop(subfield_position)<EOL>subfields.insert(new_subfield_position, subfield)<EOL><DEDENT>except IndexError:<EOL><INDENT>raise InvenioBibRecordFieldError(<EOL>"<STR_LIT>" % subfield_position)<EOL><DEDENT>

Move subfield at specified position.

    Sspecify the subfield by tag, field number and subfield position to new
    subfield position.


def record_get_field_value(rec, tag, ind1="<STR_LIT:U+0020>", ind2="<STR_LIT:U+0020>", code="<STR_LIT>"):

<EOL>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>if '<STR_LIT:%>' in tag:<EOL><INDENT>if code == '<STR_LIT>':<EOL><INDENT>for field_tag, fields in list(rec.items()):<EOL><INDENT>if _tag_matches_pattern(field_tag, tag):<EOL><INDENT>for field in fields:<EOL><INDENT>if ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]):<EOL><INDENT>if field[<NUM_LIT:3>]:<EOL><INDENT>return field[<NUM_LIT:3>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>elif code == '<STR_LIT:%>':<EOL><INDENT>for field_tag, fields in list(rec.items()):<EOL><INDENT>if _tag_matches_pattern(field_tag, tag):<EOL><INDENT>for field in fields:<EOL><INDENT>if (ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and<EOL>ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]) and field[<NUM_LIT:0>]):<EOL><INDENT>return field[<NUM_LIT:0>][<NUM_LIT:0>][<NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for field_tag, fields in list(rec.items()):<EOL><INDENT>if _tag_matches_pattern(field_tag, tag):<EOL><INDENT>for field in fields:<EOL><INDENT>if ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]):<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == code:<EOL><INDENT>return subfield[<NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>if tag in rec:<EOL><INDENT>if code == '<STR_LIT>':<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>if ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]):<EOL><INDENT>if field[<NUM_LIT:3>]:<EOL><INDENT>return field[<NUM_LIT:3>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif code == '<STR_LIT:%>':<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>if ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]) andfield[<NUM_LIT:0>]:<EOL><INDENT>return field[<NUM_LIT:0>][<NUM_LIT:0>][<NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>if ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]):<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == code:<EOL><INDENT>return subfield[<NUM_LIT:1>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return "<STR_LIT>"<EOL>

Return first (string) value that matches specified field of the record.

    Returns empty string if not found.

    Parameters (tag, ind1, ind2, code) can contain wildcard %.

    Difference between wildcard % and empty '':

    - Empty char specifies that we are not interested in a field which
      has one of the indicator(s)/subfield specified.

    - Wildcard specifies that we are interested in getting the value
      of the field whatever the indicator(s)/subfield is.

    For e.g. consider the following record in MARC::

        100C5  $$a val1
        555AB  $$a val2
        555AB      val3
        555    $$a val4
        555A       val5

    .. doctest::

        >>> record_get_field_value(record, '555', 'A', '', '')
        "val5"
        >>> record_get_field_value(record, '555', 'A', '%', '')
        "val3"
        >>> record_get_field_value(record, '555', 'A', '%', '%')
        "val2"
        >>> record_get_field_value(record, '555', 'A', 'B', '')
        "val3"
        >>> record_get_field_value(record, '555', '', 'B', 'a')
        ""
        >>> record_get_field_value(record, '555', '', '', 'a')
        "val4"
        >>> record_get_field_value(record, '555', '', '', '')
        ""
        >>> record_get_field_value(record, '%%%', '%', '%', '%')
        "val1"

    :param rec: a record structure as returned by create_record()
    :param tag: a 3 characters long string
    :param ind1: a 1 character long string
    :param ind2: a 1 character long string
    :param code: a 1 character long string
    :return: string value (empty if nothing found)


tmp = []<EOL>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>if filter_subfield_code and filter_subfield_mode == "<STR_LIT:r>":<EOL><INDENT>reg_exp = re.compile(filter_subfield_value)<EOL><DEDENT>tags = []<EOL>if '<STR_LIT:%>' in tag:<EOL><INDENT>tags = [k for k in rec if _tag_matches_pattern(k, tag)]<EOL><DEDENT>elif rec and tag in rec:<EOL><INDENT>tags = [tag]<EOL><DEDENT>if code == '<STR_LIT>':<EOL><INDENT>for tag in tags:<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>if (ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]) and<EOL>field[<NUM_LIT:3>]):<EOL><INDENT>tmp.append(field[<NUM_LIT:3>])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif code == '<STR_LIT:%>':<EOL><INDENT>for tag in tags:<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>if ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]):<EOL><INDENT>if filter_subfield_code:<EOL><INDENT>if filter_subfield_mode == "<STR_LIT:e>":<EOL><INDENT>subfield_to_match = (filter_subfield_code,<EOL>filter_subfield_value)<EOL>if subfield_to_match in field[<NUM_LIT:0>]:<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>tmp.append(subfield[<NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT>elif filter_subfield_mode == "<STR_LIT:s>":<EOL><INDENT>if (dict(field[<NUM_LIT:0>]).get(filter_subfield_code, '<STR_LIT>')).find(filter_subfield_value) > -<NUM_LIT:1>:<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>tmp.append(subfield[<NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT>elif filter_subfield_mode == "<STR_LIT:r>":<EOL><INDENT>if reg_exp.match(dict(field[<NUM_LIT:0>])<EOL>.get(filter_subfield_code, '<STR_LIT>')):<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>tmp.append(subfield[<NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>tmp.append(subfield[<NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for tag in tags:<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>if ind1 in ('<STR_LIT:%>', field[<NUM_LIT:1>]) and ind2 in ('<STR_LIT:%>', field[<NUM_LIT:2>]):<EOL><INDENT>if filter_subfield_code:<EOL><INDENT>if filter_subfield_mode == "<STR_LIT:e>":<EOL><INDENT>subfield_to_match = (filter_subfield_code,<EOL>filter_subfield_value)<EOL>if subfield_to_match in field[<NUM_LIT:0>]:<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == code:<EOL><INDENT>tmp.append(subfield[<NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif filter_subfield_mode == "<STR_LIT:s>":<EOL><INDENT>if (dict(field[<NUM_LIT:0>]).get(filter_subfield_code, '<STR_LIT>')).find(filter_subfield_value) > -<NUM_LIT:1>:<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == code:<EOL><INDENT>tmp.append(subfield[<NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT><DEDENT>elif filter_subfield_mode == "<STR_LIT:r>":<EOL><INDENT>if reg_exp.match(dict(field[<NUM_LIT:0>])<EOL>.get(filter_subfield_code, '<STR_LIT>')):<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == code:<EOL><INDENT>tmp.append(subfield[<NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>else:<EOL><INDENT>for subfield in field[<NUM_LIT:0>]:<EOL><INDENT>if subfield[<NUM_LIT:0>] == code:<EOL><INDENT>tmp.append(subfield[<NUM_LIT:1>])<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return tmp<EOL>

Return the list of values for the specified field of the record.

    List can be filtered. Use filter_subfield_code
    and filter_subfield_value to search
    only in fields that have these values inside them as a subfield.
    filter_subfield_mode can have 3 different values:
    'e' for exact search
    's' for substring search
    'r' for regexp search

    Returns empty list if nothing was found.

    Parameters (tag, ind1, ind2, code) can contain wildcard %.

    :param rec: a record structure as returned by create_record()
    :param tag: a 3 characters long string
    :param ind1: a 1 character long string
    :param ind2: a 1 character long string
    :param code: a 1 character long string
    :return: a list of strings


def record_xml_output(rec, tags=None, order_fn=None):

if tags is None:<EOL><INDENT>tags = []<EOL><DEDENT>if isinstance(tags, str):<EOL><INDENT>tags = [tags]<EOL><DEDENT>if tags and '<STR_LIT>' not in tags:<EOL><INDENT>tags.append('<STR_LIT>')<EOL><DEDENT>marcxml = ['<STR_LIT>']<EOL>fields = []<EOL>if rec is not None:<EOL><INDENT>for tag in rec:<EOL><INDENT>if not tags or tag in tags:<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>fields.append((tag, field))<EOL><DEDENT><DEDENT><DEDENT>if order_fn is None:<EOL><INDENT>record_order_fields(fields)<EOL><DEDENT>else:<EOL><INDENT>record_order_fields(fields, order_fn)<EOL><DEDENT>for field in fields:<EOL><INDENT>marcxml.append(field_xml_output(field[<NUM_LIT:1>], field[<NUM_LIT:0>]))<EOL><DEDENT><DEDENT>marcxml.append('<STR_LIT>')<EOL>return '<STR_LIT:\n>'.join(marcxml)<EOL>

Generate the XML for record 'rec'.

    :param rec: record
    :param tags: list of tags to be printed
    :return: string


return field[<NUM_LIT:0>]<EOL>

Return the list of subfields associated with field 'field'.


return [subfield_value<EOL>for subfield_code, subfield_value in field_instance[<NUM_LIT:0>]<EOL>if subfield_code == code]<EOL>

Return subfield CODE values of the field instance FIELD.


return [subfield_code<EOL>for subfield_code, subfield_value in field_instance[<NUM_LIT:0>]]<EOL>

Return subfield codes of the field instance FIELD.


def field_add_subfield(field, code, value):

field[<NUM_LIT:0>].append((code, value))<EOL>


rec.sort(eval(fun))<EOL>

Order field inside record 'rec' according to a function.


marcxml = []<EOL>if field[<NUM_LIT:3>]:<EOL><INDENT>marcxml.append('<STR_LIT>' %<EOL>(tag, MathMLParser.html_to_text(field[<NUM_LIT:3>])))<EOL><DEDENT>else:<EOL><INDENT>marcxml.append('<STR_LIT>' %<EOL>(tag, field[<NUM_LIT:1>], field[<NUM_LIT:2>]))<EOL>marcxml += [_subfield_xml_output(subfield) for subfield in field[<NUM_LIT:0>]]<EOL>marcxml.append('<STR_LIT>')<EOL><DEDENT>return '<STR_LIT:\n>'.join(marcxml)<EOL>

Generate the XML for field 'field' and returns it as a string.


record_dois = []<EOL>tag = "<STR_LIT>"<EOL>ind1 = "<STR_LIT>"<EOL>ind2 = "<STR_LIT:_>"<EOL>subfield_source_code = "<STR_LIT:2>"<EOL>subfield_value_code = "<STR_LIT:a>"<EOL>identifiers_fields = record_get_field_instances(record, tag, ind1, ind2)<EOL>for identifer_field in identifiers_fields:<EOL><INDENT>if '<STR_LIT>' in [val.lower() for val in<EOL>field_get_subfield_values(identifer_field,<EOL>subfield_source_code)]:<EOL><INDENT>record_dois.extend(<EOL>field_get_subfield_values(<EOL>identifer_field,<EOL>subfield_value_code))<EOL><DEDENT><DEDENT>return record_dois<EOL>


def print_rec(rec, format=<NUM_LIT:1>, tags=None):

if tags is None:<EOL><INDENT>tags = []<EOL><DEDENT>if format == <NUM_LIT:1>:<EOL><INDENT>text = record_xml_output(rec, tags)<EOL><DEDENT>else:<EOL><INDENT>return '<STR_LIT>'<EOL><DEDENT>return text<EOL>

Print a record.

:param format: 1 XML, 2 HTML (not implemented)
:param tags: list of tags to be printed


def print_recs(listofrec, format=<NUM_LIT:1>, tags=None):

if tags is None:<EOL><INDENT>tags = []<EOL><DEDENT>text = "<STR_LIT>"<EOL>if type(listofrec).__name__ != '<STR_LIT:list>':<EOL><INDENT>return "<STR_LIT>"<EOL><DEDENT>else:<EOL><INDENT>for rec in listofrec:<EOL><INDENT>text = "<STR_LIT>" % (text, print_rec(rec, format, tags))<EOL><DEDENT><DEDENT>return text<EOL>

Print a list of records.

:param format: 1 XML, 2 HTML (not implemented)
:param tags: list of tags to be printed
       if 'listofrec' is not a list it returns empty string


newl = []<EOL>for l in alist:<EOL><INDENT>newl.extend(l)<EOL><DEDENT>return newl<EOL>


def record_find_field(rec, tag, field, strict=False):

try:<EOL><INDENT>_check_field_validity(field)<EOL><DEDENT>except InvenioBibRecordFieldError:<EOL><INDENT>raise<EOL><DEDENT>for local_position, field1 in enumerate(rec.get(tag, [])):<EOL><INDENT>if _compare_fields(field, field1, strict):<EOL><INDENT>return (field1[<NUM_LIT:4>], local_position)<EOL><DEDENT><DEDENT>return (None, None)<EOL>

Return the global and local positions of the first occurrence of the field.

:param rec:    A record dictionary structure
:type  rec:    dictionary
:param tag:    The tag of the field to search for
:type  tag:    string
:param field:  A field tuple as returned by create_field()
:type  field:  tuple
:param strict: A boolean describing the search method. If strict
               is False, then the order of the subfields doesn't
               matter. Default search method is strict.
:type  strict: boolean
:return:       A tuple of (global_position, local_position) or a
               tuple (None, None) if the field is not present.
:rtype:        tuple
:raise InvenioBibRecordFieldError: If the provided field is invalid.


def record_match_subfields(rec, tag, ind1="<STR_LIT:U+0020>", ind2="<STR_LIT:U+0020>", sub_key=None,<EOL>sub_value='<STR_LIT>', sub_key2=None, sub_value2='<STR_LIT>',<EOL>case_sensitive=True):

if sub_key is None:<EOL><INDENT>raise TypeError("<STR_LIT>")<EOL><DEDENT>if sub_key2 is not None and sub_value2 is '<STR_LIT>':<EOL><INDENT>raise TypeError("<STR_LIT>"<EOL>+ "<STR_LIT>")<EOL><DEDENT>ind1, ind2 = _wash_indicators(ind1, ind2)<EOL>if not case_sensitive:<EOL><INDENT>sub_value = sub_value.lower()<EOL>sub_value2 = sub_value2.lower()<EOL><DEDENT>for field in record_get_field_instances(rec, tag, ind1, ind2):<EOL><INDENT>subfields = dict(field_get_subfield_instances(field))<EOL>if not case_sensitive:<EOL><INDENT>for k, v in subfields.items():<EOL><INDENT>subfields[k] = v.lower()<EOL><DEDENT><DEDENT>if sub_key in subfields:<EOL><INDENT>if sub_value is '<STR_LIT>':<EOL><INDENT>return field[<NUM_LIT:4>]<EOL><DEDENT>else:<EOL><INDENT>if sub_value == subfields[sub_key]:<EOL><INDENT>if sub_key2 is None:<EOL><INDENT>return field[<NUM_LIT:4>]<EOL><DEDENT>else:<EOL><INDENT>if sub_key2 in subfields:<EOL><INDENT>if sub_value2 == subfields[sub_key2]:<EOL><INDENT>return field[<NUM_LIT:4>]<EOL><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT><DEDENT>return False<EOL>

Find subfield instances in a particular field.

It tests values in 1 of 3 possible ways:
 - Does a subfield code exist? (ie does 773__a exist?)
 - Does a subfield have a particular value? (ie 773__a == 'PhysX')
 - Do a pair of subfields have particular values?
    (ie 035__2 == 'CDS' and 035__a == '123456')

Parameters:
 * rec - dictionary: a bibrecord structure
 * tag - string: the tag of the field (ie '773')
 * ind1, ind2 - char: a single characters for the MARC indicators
 * sub_key - char: subfield key to find
 * sub_value - string: subfield value of that key
 * sub_key2 - char: key of subfield to compare against
 * sub_value2 - string: expected value of second subfield
 * case_sensitive - bool: be case sensitive when matching values

:return: false if no match found, else provides the field position (int)


for tag in list(rec.keys()):<EOL><INDENT>for field in rec[tag]:<EOL><INDENT>field[<NUM_LIT:0>][:] = [subfield for subfield in field[<NUM_LIT:0>]<EOL>if subfield[<NUM_LIT:1>][:<NUM_LIT:9>] != "<STR_LIT>"]<EOL><DEDENT><DEDENT>

Remove unchanged volatile subfields from the record.


for tag in list(rec.keys()):<EOL><INDENT>for field_position, field in enumerate(rec[tag]):<EOL><INDENT>for subfield_position, subfield in enumerate(field[<NUM_LIT:0>]):<EOL><INDENT>if subfield[<NUM_LIT:1>][:<NUM_LIT:9>] != "<STR_LIT>":<EOL><INDENT>record_modify_subfield(rec, tag, subfield[<NUM_LIT:0>], "<STR_LIT>" + subfield[<NUM_LIT:1>],<EOL>subfield_position, field_position_local=field_position)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>