Repository: RedHatInsights/insights-core

Issue:
Need to enhance the keep_scan and token_scan of LogFileOutput
Currently, the 'keep_scan' and 'token_scan' of "LogFileOutput" has supported scan a list of strings which works to search all 'items' in the list in one line.

After investigating all the usage of self-defined "_scan" in insights-rules, I found some common usage:

- Get the 'last' one or few (specific number (5 or 10)) of impacted lines from the log file.
- Check if there are some lines contain 'any' of the special 'keyword' s

So, this Card aims to enhance these '*_scan' s with following features:

- Besides the 'all' checking feature, add a 'any' checking feature against the list for both 'keep_scan' and 'token_scan'

  User can chose the 'all' and 'any' as they want
  User can specify the max 'number' of lines then want to get 
  User can chose starting the scan from the 'head' or 'tail'
- Add an individual interface for the 'last' line.



Discussion:

Resolving patch:
diff --git a/insights/core/__init__.py b/insights/core/__init__.py
--- a/insights/core/__init__.py
+++ b/insights/core/__init__.py
@@ -859,8 +859,13 @@ class LogFileOutput(six.with_metaclass(ScanMeta, Parser)):
         ...     pass
         >>> MyLogger.keep_scan('get_one', 'one')
         >>> MyLogger.keep_scan('get_three_and_more', ['three', 'more'])
-        >>> MyLogger.token_scan('is_more', 'more')
-        >>> MyLogger.token_scan('is_more_and_more', ['four', 'more'])
+        >>> MyLogger.keep_scan('get_one_or_two', ['one', 'two'], check=any)
+        >>> MyLogger.last_scan('last_line_contains_file', 'file')
+        >>> MyLogger.keep_scan('last_2_lines_contain_file', 'file', num=2, reverse=True)
+        >>> MyLogger.keep_scan('last_3_lines_contain_line_and_t', ['line', 't'], num=3, reverse=True)
+        >>> MyLogger.token_scan('find_more', 'more')
+        >>> MyLogger.token_scan('find_four_and_more', ['four', 'more'])
+        >>> MyLogger.token_scan('find_four_or_more', ['four', 'more'], check=any)
         >>> my_logger = MyLogger(context_wrap(contents, path='/var/log/mylog'))
         >>> my_logger.file_path
         '/var/log/mylog'
@@ -868,7 +873,7 @@ class LogFileOutput(six.with_metaclass(ScanMeta, Parser)):
         'mylog'
         >>> my_logger.get('two')
         [{'raw_message': 'Log file line two'}]
-        >>> 'three' in my_logger
+        >>> 'line three,' in my_logger
         True
         >>> my_logger.get(['three', 'more'])
         [{'raw_message': 'Log file line three, and more'}]
@@ -878,10 +883,18 @@ class LogFileOutput(six.with_metaclass(ScanMeta, Parser)):
         [{'raw_message': 'Log file line one'}]
         >>> my_logger.get_three_and_more == my_logger.get(['three', 'more'])
         True
-        >>> my_logger.is_more
+        >>> my_logger.last_line_contains_file
+        {'raw_message': 'Log file line three, and more'}
+        >>> len(my_logger.last_2_lines_contain_file)
+        2
+        >>> len(my_logger.last_3_lines_contain_line_and_t)  # Only 2 lines contain 'line' and 't'
+        2
+        >>> my_logger.find_more
         True
-        >>> my_logger.is_more_and_more
+        >>> my_logger.find_four_and_more
         False
+        >>> my_logger.find_four_or_more
+        True
 
     Attributes:
         lines (list): List of the lines from the log file content.
@@ -910,7 +923,8 @@ def parse_content(self, content):
 
     def __contains__(self, s):
         """
-        Returns true if any line contains the given text string.
+        Return ``True`` if any line contains the given text string or all the
+        strings in the given list.
         """
         search_by_expression = self._valid_search(s)
         return any(search_by_expression(l) for l in self.lines)
@@ -922,7 +936,7 @@ def _parse_line(self, line):
         """
         return {'raw_message': line}
 
-    def _valid_search(self, s):
+    def _valid_search(self, s, check=all):
         """
         Check this given `s`, it must be a string or a list of strings.
         Otherwise, a TypeError will be raised.
@@ -931,29 +945,39 @@ def _valid_search(self, s):
             return lambda l: s in l
         elif (isinstance(s, list) and len(s) > 0 and
               all(isinstance(w, six.string_types) for w in s)):
-            return lambda l: all(w in l for w in s)
+            return lambda l: check(w in l for w in s)
         elif s is not None:
             raise TypeError('Search items must be given as a string or a list of strings')
 
-    def get(self, s):
+    def get(self, s, check=all, num=None, reverse=False):
         """
         Returns all lines that contain `s` anywhere and wrap them in a list of
         dictionaries.  `s` can be either a single string or a string list. For
         list, all keywords in the list must be found in each line.
 
         Parameters:
-            s(str or list): one or more strings to search for.
+            s(str or list): one or more strings to search for
+            check(func): built-in function ``all`` or ``any`` applied to each line
+            num(int): the number of lines to get, ``None`` for unlimited
+            reverse(bool): scan start from the head when ``False`` by default, otherwise start from the tail
 
         Returns:
-            (list): list of dictionaries corresponding to the parsed lines
-            contain the `s`.
+            (list): list of dictionaries corresponding to the parsed lines contain the `s`.
+
+        Raises:
+            TypeError: When `s` is not a string or a list of strings, or `num`
+                is not an integer.
         """
+        if num is not None and not isinstance(num, six.integer_types):
+            raise TypeError('Required numbers must be given as a integer')
         ret = []
-        search_by_expression = self._valid_search(s)
-        for l in self.lines:
-            if search_by_expression(l):
+        search_by_expression = self._valid_search(s, check)
+        lines = self.lines[::-1] if reverse else self.lines
+        for l in lines:
+            if (num is None or len(ret) < num) and search_by_expression(l):
                 ret.append(self._parse_line(l))
-        return ret
+        # re-sort to original order
+        return ret[::-1] if reverse else ret
 
     @classmethod
     def scan(cls, result_key, func):
@@ -961,6 +985,9 @@ def scan(cls, result_key, func):
         Define computed fields based on a string to "grep for".  This is
         preferred to utilizing raw log lines in plugins because computed fields
         will be serialized, whereas raw log lines will not.
+
+        Raises:
+            ValueError: When `result_key` is already a registered scanner key.
         """
 
         if result_key in cls.scanner_keys:
@@ -974,24 +1001,65 @@ def scanner(self):
         cls.scanner_keys.add(result_key)
 
     @classmethod
-    def token_scan(cls, result_key, token):
+    def token_scan(cls, result_key, token, check=all, reverse=False):
         """
         Define a property that is set to true if the given token is found in
         the log file.  Uses the __contains__ method of the log file.
+
+        Parameters:
+            result_key(str): the scanner key to register
+            token(str or list): one or more strings to search for
+            check(func): built-in function ``all`` or ``any`` applied to each line
+            reverse(bool): scan start from the head when ``False`` by default, otherwise start from the tail
+
+        Returns:
+            (list): list of dictionaries corresponding to the parsed lines contain the `token`.
+        """
+        def _scan(self):
+            search_by_expression = self._valid_search(token, check)
+            lines = self.lines[::-1] if reverse else self.lines
+            return any(search_by_expression(l) for l in lines)
+
+        cls.scan(result_key, _scan)
+
+    @classmethod
+    def keep_scan(cls, result_key, token, check=all, num=None, reverse=False):
+        """
+        Define a property that is set to the list of dictionaries of the lines
+        that contain the given token.  Uses the get method of the log file.
+
+        Parameters:
+            result_key(str): the scanner key to register
+            token(str or list): one or more strings to search for
+            check(func): built-in function ``all`` or ``any`` applied to each line
+            num(int): the number of lines to get, ``None`` for unlimited
+            reverse(bool): scan start from the head when ``False`` by default, otherwise start from the tail
+
+        Returns:
+            (list): list of dictionaries corresponding to the parsed lines contain the `token`.
         """
         def _scan(self):
-            return token in self
+            return self.get(token, check=check, num=num, reverse=reverse)
 
         cls.scan(result_key, _scan)
 
     @classmethod
-    def keep_scan(cls, result_key, token):
+    def last_scan(cls, result_key, token, check=all):
         """
-        Define a property that is set to the list of lines that contain the
-        given token.  Uses the get method of the log file.
+        Define a property that is set to the dictionary of the last line
+        that contains the given token.  Uses the get method of the log file.
+
+        Parameters:
+            result_key(str): the scanner key to register
+            token(str or list): one or more strings to search for
+            check(func): built-in function ``all`` or ``any`` applied to each line
+
+        Returns:
+            (dict): dictionary corresponding to the last parsed line contains the `token`.
         """
         def _scan(self):
-            return self.get(token)
+            ret = self.get(token, check=check, num=1, reverse=True)
+            return ret[0] if ret else dict()
 
         cls.scan(result_key, _scan)
 


Test patch:
diff --git a/insights/tests/test_doc_examples.py b/insights/tests/test_doc_examples.py
new file mode 100644
diff --git a/insights/tests/test_logfileoutput.py b/insights/tests/test_logfileoutput.py
--- a/insights/tests/test_logfileoutput.py
+++ b/insights/tests/test_logfileoutput.py
@@ -33,7 +33,7 @@ def listget(self, s):
 
 
 MESSAGES = """
-Mar 27 03:18:15 system rsyslogd: [origin software="rsyslogd" swVersion="5.8.10" x-pid="1870" x-info="http://www.rsyslog.com"] rsyslogd was HUPed
+Mar 27 03:18:15 system rsyslogd: [origin software="rsyslogd" swVersion="5.8.10" x-pid="1870" x-info="http://www.rsyslog.com"] first
 Mar 27 03:18:16 system rsyslogd-2177: imuxsock lost 141 messages from pid 55082 due to rate-limiting
 Mar 27 03:18:19 system rsyslogd-2177: imuxsock begins to drop messages from pid 55082 due to rate-limiting
 Mar 27 03:18:21 system pulp: pulp.server.db.connection:INFO: Attempting Database connection with seeds = localhost:27017
@@ -114,9 +114,19 @@ def test_messages_scanners_list():
     # Messages that are present can be kept
     FakeMessagesClass.keep_scan('puppet_master_manifest_logs', ['puppet-master', 'manifest'])
     FakeMessagesClass.keep_scan('puppet_master_first', ['puppet-master', 'first'])
+    # Messages that are present can be kept for any of the list
+    FakeMessagesClass.keep_scan('puppet_master_first_any', ['puppet-master', 'first'], check=any)
+    FakeMessagesClass.keep_scan('puppet_master_first_any_3', ['puppet-master', 'first'], num=3, check=any)
     # Token scan of something that's absent
     FakeMessagesClass.token_scan('error_missing', ['ERROR', 'Missing'])
+    # Token scan of something that's absent
     FakeMessagesClass.token_scan('error_info', ['ERROR', 'info'])
+    # Token scan of something that's absent for any of the list
+    FakeMessagesClass.token_scan('error_info_any', ['ERROR', 'info'], check=any)
+    # Get the last line
+    FakeMessagesClass.last_scan('puppet_master_manifest_last', ['puppet-master', 'manifest'])
+    # Get the last line for any of the list
+    FakeMessagesClass.last_scan('puppet_master_first_last_any', ['puppet-master', 'first'], check=any)
 
     ctx = context_wrap(MESSAGES, path='/var/log/messages')
     log = FakeMessagesClass(ctx)
@@ -131,6 +141,19 @@ def test_messages_scanners_list():
     assert hasattr(log, 'error_info')
     assert log.error_info is False
 
+    assert hasattr(log, 'puppet_master_first_any')
+    assert len(log.puppet_master_first_any) == 7
+    assert hasattr(log, 'puppet_master_first_any_3')
+    assert len(log.puppet_master_first_any_3) == 3
+    assert hasattr(log, 'error_info_any')
+    assert log.error_info_any is True
+
+    assert hasattr(log, 'puppet_master_manifest_last')
+    assert 'puppet-master' in log.puppet_master_manifest_last['raw_message']
+    assert 'manifest' in log.puppet_master_manifest_last['raw_message']
+    assert hasattr(log, 'puppet_master_first_last_any')
+    assert 'puppet-master' in log.puppet_master_first_last_any['raw_message']
+
 
 def test_messages_get_after():
     ctx = context_wrap(MESSAGES, path='/var/log/messages')