Repository: RedHatInsights/insights-core

Issue:
Create a PS combiner to consolidate data from PS parsers 
Create a PS combiner that consolidates processes data (PS command based) from all the PS parsers. The combiner should have unified interface and standardized set of data attributes.  


Discussion:

Resolving patch:
diff --git a/insights/combiners/ps.py b/insights/combiners/ps.py
new file mode 100644
--- /dev/null
+++ b/insights/combiners/ps.py
@@ -0,0 +1,300 @@
+"""
+PS
+==
+
+This combiner provides information about running processes based on the ``ps`` command.
+More specifically this consolidates data from
+:py:class:`insights.parsers.ps.PsEo`,
+:py:class:`insights.parsers.ps.PsAuxcww`,
+:py:class:`insights.parsers.ps.PsEf`,
+:py:class:`insights.parsers.ps.PsAux`,
+:py:class:`insights.parsers.ps.PsAuxww` and
+:py:class:`insights.parsers.ps.PsAlxwww` parsers (in that specific order).
+
+Note:
+    The final dataset can vary depending on availability of the parsers for a given
+    ``ExecutionContext`` and added filters. The underlying filterable datasources
+    for this combiner can be filtered by passing :py:class:`insights.combiners.ps.Ps`
+    to :py:func:`insights.core.filters.add_filter` function along with a filter pattern.
+    Please see :py:mod:`insights.core.filters` for more information on filtering.
+
+
+Examples:
+
+    >>> ps_combiner.pids
+    [1, 2, 3, 8, 9, 10, 11, 12]
+    >>> '[kthreadd]' in ps_combiner.commands
+    True
+    >>> '[kthreadd]' in ps_combiner
+    True
+    >>> ps_combiner[2] == {
+    ... 'PID': 2,
+    ... 'USER': 'root',
+    ... 'UID': 0,
+    ... 'PPID': 0,
+    ... '%CPU': 0.0,
+    ... '%MEM': 0.0,
+    ... 'VSZ': 0.0,
+    ... 'RSS': 0.0,
+    ... 'TTY': '?',
+    ... 'STAT': 'S',
+    ... 'START': '2019',
+    ... 'TIME': '1:04',
+    ... 'COMMAND': '[kthreadd]',
+    ... 'COMMAND_NAME': '[kthreadd]',
+    ... 'ARGS': '',
+    ... 'F': '1',
+    ... 'PRI': 20,
+    ... 'NI': '0',
+    ... 'WCHAN': 'kthrea'
+    ... }
+    True
+"""
+
+from insights.core.plugins import combiner
+from insights.parsers import keyword_search
+from insights.parsers.ps import PsAlxwww, PsAuxww, PsAux, PsAuxcww, PsEo, PsEf
+
+
+@combiner([PsAlxwww, PsAuxww, PsAux, PsEf, PsAuxcww, PsEo])
+class Ps(object):
+    """
+    ``Ps`` combiner consolidates data from the parsers in ``insights.parsers.ps`` module.
+    """
+
+    # data attributes type conversion map
+    __CONVERSION_MAP = {
+        'PID': int,
+        'UID': int,
+        'PPID': int,
+        '%CPU': float,
+        '%MEM': float,
+        'VSZ': float,
+        'RSS': float,
+        'PRI': int
+    }
+
+    # empty base row
+    __EMPTY_ROW = {
+        'PID': None,
+        'USER': None,
+        'UID': None,
+        'PPID': None,
+        '%CPU': None,
+        '%MEM': None,
+        'VSZ': None,
+        'RSS': None,
+        'TTY': None,
+        'STAT': None,
+        'START': None,
+        'TIME': None,
+        'COMMAND': None,
+        'COMMAND_NAME': None,
+        'ARGS': None,
+        'F': None,
+        'PRI': None,
+        'NI': None,
+        'WCHAN': None
+    }
+
+    def __init__(self, ps_alxwww, ps_auxww, ps_aux, ps_ef, ps_auxcww, ps_eo):
+        self._pid_data = {}
+
+        # order of parsers is important here
+        if ps_eo:
+            self.__update_data(ps_eo)
+        if ps_auxcww:
+            self.__update_data(ps_auxcww)
+        if ps_ef:
+            # mapping configurations to combine PsEf data
+            mapping = {
+                'UID': ('USER', True),
+                # cpu value (integer value) from PsEf parser should not override an existing value
+                'C': ('%CPU', False),
+                'CMD': ('COMMAND', True),
+                'STIME': ('START', True)
+            }
+            self.__update_data(ps_ef, mapping)
+        if ps_aux:
+            self.__update_data(ps_aux)
+        if ps_auxww:
+            self.__update_data(ps_auxww)
+        if ps_alxwww:
+            self.__update_data(ps_alxwww)
+
+        self.__convert_data_types()
+
+        self._commands = set(row['COMMAND'] for row in self._pid_data.values())
+
+    @property
+    def pids(self):
+        """
+        Returns the list of running process IDs (integers).
+
+
+        Returns:
+            list: the PIDs from the PID column.
+        """
+        return list(self._pid_data.keys())
+
+    @property
+    def processes(self):
+        """
+        Returns the list of dictionaries, where each item in the list represents a process
+        and the keys in each dictionary are the column headers.
+
+        Returns:
+            list: the list of running processes.
+        """
+        return list(self._pid_data.values())
+
+    @property
+    def commands(self):
+        """
+        Returns the set of full command strings for each command
+        including optional path and arguments, unless underlying parser
+        contains command names only.
+
+        Returns:
+            set: the set with command strings.
+        """
+        return self._commands
+
+    def search(self, **kwargs):
+        """
+        Search the process list for matching rows based on key-value pairs.
+
+        This uses the :py:func:`insights.parsers.keyword_search` function for
+        searching; see its documentation for usage details.  If no search
+        parameters are given, no rows are returned.
+
+        Returns:
+            list: A list of dictionaries of processes that match the given
+            search criteria.
+
+        Examples:
+            >>> ps_combiner.search(COMMAND__contains='[rcu_bh]') == [
+            ... {'PID': 9, 'USER': 'root', 'UID': 0, 'PPID': 2, '%CPU': 0.1, '%MEM': 0.0,
+            ...  'VSZ': 0.0, 'RSS': 0.0, 'TTY': '?', 'STAT': 'S', 'START': '2019', 'TIME': '0:00',
+            ...  'COMMAND': '[rcu_bh]', 'COMMAND_NAME': '[rcu_bh]', 'ARGS': '', 'F': '1', 'PRI': 20,
+            ...  'NI': '0', 'WCHAN': 'rcu_gp'}
+            ... ]
+            True
+            >>> ps_combiner.search(USER='root', COMMAND='[kthreadd]') == [
+            ... {'PID': 2, 'USER': 'root', 'UID': 0, 'PPID': 0, '%CPU': 0.0, '%MEM': 0.0,
+            ...  'VSZ': 0.0, 'RSS': 0.0, 'TTY': '?', 'STAT': 'S', 'START': '2019', 'TIME': '1:04',
+            ...  'COMMAND': '[kthreadd]', 'COMMAND_NAME': '[kthreadd]', 'ARGS': '', 'F': '1', 'PRI': 20,
+            ...  'NI': '0', 'WCHAN': 'kthrea'}
+            ... ]
+            True
+        """
+        return keyword_search(self._pid_data.values(), **kwargs)
+
+    def __contains__(self, command):
+        """
+        Check if a specific command is running.
+
+        Args:
+            command (str): a command string.
+
+        Returns:
+            bool: True if command is running, otherwise False.
+        """
+        return command in self._commands
+
+    def __getitem__(self, pid):
+        """
+        Retrieve a specific process by its PID.
+
+        Args:
+            pid (int): process ID integer value.
+
+        Returns:
+            dict: dictionary that represents a process with each key
+                as the column header.
+        """
+        return self._pid_data.get(pid)
+
+    def __iter__(self):
+        for row in self._pid_data.values():
+            yield row
+
+    def __update_data(self, ps_parser, mapping=None):
+        """
+        Updates internal dictionary with the processes data from the parser.
+        New PIDs will be add added to the dictionary and existing ones
+        will be updated. ``mapping`` needs to specify attribute mapping
+        metadata for proper consolidation of data.
+
+        Args:
+            ps_parser (insights.parsers.ps.Ps): Ps parser implementation instance.
+            mapping (dict): parser data mapping configurations.
+
+        Returns:
+            None
+        """
+        def update_row(input_row, mapping):
+            pid = int(input_row['PID'])
+            # if new PID then add base empty row to the dictionary using `setdefault`
+            pid_row = self._pid_data.setdefault(pid, self.__EMPTY_ROW.copy())
+            temp_row = self.__map_row(pid, input_row, mapping)
+            pid_row.update(temp_row)
+
+        [update_row(row, mapping)
+        for row in ps_parser.data]
+
+    def __convert_data_types(self):
+        """
+        Convert types in the final dataset for attributes defined
+        in ``__CONVERSION_MAP`` dictionary.
+
+        Returns:
+            None
+        """
+        def convert_attr(attr_name, row):
+            if row[attr_name] is not None:
+                type_ctor = self.__CONVERSION_MAP[attr_name]
+                row[attr_name] = type_ctor(row[attr_name])
+
+        [convert_attr(attr_name, row)
+        for attr_name in self.__CONVERSION_MAP
+        for row in self._pid_data.values()
+        if attr_name in row]
+
+    def __map_row(self, pid, row, mapping):
+        """
+        Creates new data row based on provided mapping configurations
+        in ``mapping`` dictionary. If no mappings provided, original data row
+        will be copied without any changes.
+
+        Args:
+            pid (int): a process PID.
+            row (dict): a data row dictionary from a Ps parser.
+            mapping(dict): ps data mapping configurations,
+            the format for this is:
+            ``original_attribute_name: (destination_attiribute_name, override_value_if_populated_flag)``
+
+        Returns:
+            dict: Modified data row that represents a process.
+        """
+        r_row = {}
+        if mapping:
+            for attr_name in row:
+                if attr_name in mapping:
+                    # `dest_name` is mapping destination attribute name
+                    # `override` is bool flag to identify if already populated
+                    #  attribute value should be overridden/updated
+                    dest_name, override = mapping[attr_name]
+                    # don't map attribute if destination is None
+                    if dest_name is None:
+                        continue
+                    if not override:
+                        # don't update attribute value if already populated for this PID
+                        if self._pid_data.setdefault(pid, self.__EMPTY_ROW.copy)[dest_name] is not None:
+                            continue
+                    r_row[dest_name] = row[attr_name]
+                else:
+                    r_row[attr_name] = row[attr_name]
+        else:
+            r_row = row.copy()
+        return r_row


Test patch:
diff --git a/insights/combiners/tests/test_ps.py b/insights/combiners/tests/test_ps.py
new file mode 100644
--- /dev/null
+++ b/insights/combiners/tests/test_ps.py
@@ -0,0 +1,248 @@
+from insights.combiners import ps
+from insights.combiners.ps import Ps
+from insights.parsers.ps import PsAlxwww, PsAuxww, PsAux, PsAuxcww, PsEo, PsEf
+from insights.tests import context_wrap
+import doctest
+
+
+PS_EO_LINES = """
+  PID  PPID COMMAND
+    1     0 systemd
+    2     0 kthreadd
+    3     2 ksoftirqd/0
+    8     2 migration/0
+    9     2 rcu_bh
+    10    2 rcu_sched
+ """
+
+PS_AUXCWW_LINES = """
+USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
+root         1  0.1  0.0 195712  7756 ?        Ss    2019 477:10 systemd
+root         2  0.0  0.0      0     0 ?        S     2019   1:04 kthreadd
+root         3  0.0  0.0      0     0 ?        S     2019  36:41 ksoftirqd/0
+root         8  0.0  0.0      0     0 ?        S     2019  31:50 migration/0
+root         9  0.1  0.0      0     0 ?        S     2019   0:00 rcu_bh
+root        11  0.1  0.0      0     0 ?        S     2019  05:00 watchdog/0
+"""
+
+PS_EF_LINES = """
+UID        PID  PPID  C STIME TTY          TIME CMD
+root         1     0  0  2019 ?        07:57:10 /usr/lib/systemd/systemd --switched-root --system --deserialize 21
+root         2     0  0  2019 ?        00:01:04 [kthreadd]
+root         3     2  0  2019 ?        00:36:41 [ksoftirqd/0]
+root         8     2  0  2019 ?        00:31:50 [migration/0]
+root         9     2  0  2019 ?        00:00:00 [rcu_bh]
+root        12     2  0  2019 ?        00:05:00 [watchdog/1]
+"""
+
+PS_AUX_LINES = """
+USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
+root         1  0.1  0.0 195712  7756 ?        Ss    2019 478:05 /usr/lib/systemd/systemd --switched-root --system --deserialize 21
+root         2  0.0  0.0      0     0 ?        S     2019   1:04 [kthreadd]
+root         3  0.0  0.0      0     0 ?        S     2019  36:47 [ksoftirqd/0]
+root         8  0.0  0.0      0     0 ?        S     2019  32:38 [migration/0]
+root         9  0.1  0.0      0     0 ?        S     2019   0:00 [rcu_bh]
+"""
+
+PS_AUXWW_LINES = """
+USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
+root         1  0.1  0.0 195712  7756 ?        Ss    2019 478:05 /usr/lib/systemd/systemd --switched-root --system --deserialize 21
+root         2  0.0  0.0      0     0 ?        S     2019   1:04 [kthreadd]
+root         3  0.0  0.0      0     0 ?        S     2019  36:47 [ksoftirqd/0]
+root         8  0.0  0.0      0     0 ?        S     2019  32:38 [migration/0]
+root         9  0.1  0.0      0     0 ?        S     2019   0:00 [rcu_bh]
+"""
+
+PS_ALXWWW_LINES = """
+F   UID   PID  PPID PRI  NI    VSZ   RSS WCHAN  STAT TTY        TIME COMMAND
+4     0     1     0  20   0 195712  7756 ep_pol Ss   ?        478:05 /usr/lib/systemd/systemd --switched-root --system --deserialize 21
+1     0     2     0  20   0      0     0 kthrea S    ?          1:04 [kthreadd]
+1     0     3     2  20   0      0     0 smpboo S    ?         36:47 [ksoftirqd/0]
+1     0     8     2 -100  -      0     0 smpboo S    ?         32:38 [migration/0]
+1     0     9     2  20   0      0     0 rcu_gp S    ?          0:00 [rcu_bh]
+"""
+
+
+def test_pseo_parser():
+    ps_eo = PsEo(context_wrap(PS_EO_LINES, strip=False))
+    ps = Ps(None, None, None, None, None, ps_eo)
+    assert len(ps.processes) == 6
+    proc = ps[1]
+    assert proc['USER'] is None
+    assert proc['TTY'] is None
+    assert proc['%CPU'] is None
+    assert proc['%MEM'] is None
+    assert proc['COMMAND'] == proc['COMMAND_NAME']
+
+
+def test_pseo_and_psauxcww_parsers():
+    ps_eo = PsEo(context_wrap(PS_EO_LINES, strip=False))
+    ps_auxcww = PsAuxcww(context_wrap(PS_AUXCWW_LINES))
+    ps = Ps(None, None, None, None, ps_auxcww, ps_eo)
+    assert len(ps.processes) == 7
+    proc9 = ps[9]
+    assert proc9['USER'] == 'root'
+    assert proc9['TTY'] == '?'
+    assert proc9['%CPU'] == 0.1
+    assert proc9['%MEM'] == 0.0
+    assert proc9['COMMAND'] == proc9['COMMAND_NAME']
+    proc10 = ps[10]
+    assert proc10['USER'] is None
+    assert proc10['TTY'] is None
+    assert proc10['%CPU'] is None
+    assert proc10['%MEM'] is None
+    assert proc10['COMMAND'] == proc10['COMMAND_NAME']
+
+
+def test_psef_parser():
+    ps_ef = PsEf(context_wrap(PS_EF_LINES))
+    ps = Ps(None, None, None, ps_ef, None, None)
+    len(ps.processes) == 6
+    proc = ps[1]
+    assert proc.get('UID') is None
+    assert proc.get('C') is None
+    assert proc.get('CMD') is None
+    assert proc.get('STIME') is None
+    assert proc['USER'] == 'root'
+    assert proc['%CPU'] == 0
+    assert proc['COMMAND'] == '/usr/lib/systemd/systemd --switched-root --system --deserialize 21'
+    assert proc['START'] == '2019'
+
+
+def test_psauxcww_and_ps_ef_parsers():
+    ps_auxcww = PsAuxcww(context_wrap(PS_AUXCWW_LINES))
+    ps_ef = PsEf(context_wrap(PS_EF_LINES))
+    ps = Ps(None, None, None, ps_ef, ps_auxcww, None)
+    assert len(ps.processes) == 7
+    proc1 = ps[1]
+    assert proc1['COMMAND'] == '/usr/lib/systemd/systemd --switched-root --system --deserialize 21'
+    assert proc1['COMMAND_NAME'] == 'systemd'
+    proc9 = ps[9]
+    assert proc9['PPID'] == 2
+    assert proc9['%CPU'] == 0.1
+    assert proc9['%MEM'] == 0.0
+    assert proc9['VSZ'] == 0.0
+    proc12 = ps[12]
+    assert proc12['PPID'] == 2
+    assert proc12['%CPU'] == 0.0
+    assert proc12['%MEM'] is None
+    assert proc12['VSZ'] is None
+
+
+def test_psalxwww_and_psauxww_and_psaux_parsers():
+    ps_alxwww = PsAlxwww(context_wrap(PS_ALXWWW_LINES))
+    ps_auxww = PsAuxww(context_wrap(PS_AUXWW_LINES))
+    ps_aux = PsAux(context_wrap(PS_AUX_LINES))
+    ps = Ps(ps_alxwww, ps_auxww, ps_aux, None, None, None)
+    len(ps.processes) == 5
+    ps = ps[1]
+    assert ps['PID'] == 1
+    assert ps['USER'] == 'root'
+    assert ps['UID'] == 0
+    assert ps['PPID'] == 0
+    assert ps['%CPU'] == 0.1
+    assert ps['%MEM'] == 0.0
+    assert ps['VSZ'] == 195712.0
+    assert ps['RSS'] == 7756.0
+    assert ps['STAT'] == 'Ss'
+    assert ps['TTY'] == '?'
+    assert ps['START'] == '2019'
+    assert ps['TIME'] == '478:05'
+    assert ps['COMMAND'] == '/usr/lib/systemd/systemd --switched-root --system --deserialize 21'
+    assert ps['COMMAND_NAME'] == 'systemd'
+    assert ps['F'] == '4'
+    assert ps['PRI'] == 20
+    assert ps['NI'] == '0'
+    assert ps['WCHAN'] == 'ep_pol'
+
+
+def test_psalxwww_and_psauxww_and_psaux_and_psef_and_psauxcww_and_ps_eo_parsers():
+    ps_alxwww = PsAlxwww(context_wrap(PS_ALXWWW_LINES))
+    ps_auxww = PsAuxww(context_wrap(PS_AUXWW_LINES))
+    ps_aux = PsAux(context_wrap(PS_AUX_LINES))
+    ps_ef = PsEf(context_wrap(PS_EF_LINES))
+    ps_auxcww = PsAuxcww(context_wrap(PS_AUXCWW_LINES))
+    ps_eo = PsEo(context_wrap(PS_EO_LINES, strip=False))
+    ps = Ps(ps_alxwww, ps_auxww, ps_aux, ps_ef, ps_auxcww, ps_eo)
+    len(ps.processes) == 8
+    ps = ps[1]
+    assert ps['PID'] == 1
+    assert ps['USER'] == 'root'
+    assert ps['UID'] == 0
+    assert ps['PPID'] == 0
+    assert ps['%CPU'] == 0.1
+    assert ps['%MEM'] == 0.0
+    assert ps['VSZ'] == 195712.0
+    assert ps['RSS'] == 7756.0
+    assert ps['STAT'] == 'Ss'
+    assert ps['TTY'] == '?'
+    assert ps['START'] == '2019'
+    assert ps['TIME'] == '478:05'
+    assert ps['COMMAND'] == '/usr/lib/systemd/systemd --switched-root --system --deserialize 21'
+    assert ps['COMMAND_NAME'] == 'systemd'
+    assert ps['F'] == '4'
+    assert ps['PRI'] == 20
+    assert ps['NI'] == '0'
+    assert ps['WCHAN'] == 'ep_pol'
+
+
+def test_type_conversion():
+    ps_alxwww = PsAlxwww(context_wrap(PS_ALXWWW_LINES))
+    ps_ef = PsEf(context_wrap(PS_EF_LINES))
+    ps_auxcww = PsAuxcww(context_wrap(PS_AUXCWW_LINES))
+    ps = Ps(ps_alxwww, None, None, ps_ef, ps_auxcww, None)
+    assert all(isinstance(p['PID'], int) for p in ps.processes)
+    assert all(p['UID'] is None or isinstance(p['UID'], int) for p in ps.processes)
+    assert all(p['PID'] is None or isinstance(p['PID'], int) for p in ps.processes)
+    assert all(p['%CPU'] is None or isinstance(p['%CPU'], float) for p in ps.processes)
+    assert all(p['%MEM'] is None or isinstance(p['%MEM'], float) for p in ps.processes)
+    assert all(p['VSZ'] is None or isinstance(p['VSZ'], float) for p in ps.processes)
+    assert all(p['RSS'] is None or isinstance(p['RSS'], float) for p in ps.processes)
+    assert all(p['PRI'] is None or isinstance(p['PRI'], int) for p in ps.processes)
+
+
+def test_combiner_api():
+    ps_auxcww = PsAuxcww(context_wrap(PS_AUXCWW_LINES))
+    ps = Ps(None, None, None, None, ps_auxcww, None)
+    assert ps.pids == [1, 2, 3, 8, 9, 11]
+    assert len(ps.processes) == 6
+    assert ps.processes[0]
+    assert 'systemd' in ps.commands
+    assert len(ps.search(USER='root')) == 6
+    assert 'systemd' in ps
+    assert ps[1] == {'%CPU': 0.1,
+                     '%MEM': 0.0,
+                     'ARGS': '',
+                     'COMMAND': 'systemd',
+                     'COMMAND_NAME': 'systemd',
+                     'F': None,
+                     'NI': None,
+                     'PID': 1,
+                     'PPID': None,
+                     'PRI': None,
+                     'RSS': 7756.0,
+                     'START': '2019',
+                     'STAT': 'Ss',
+                     'TIME': '477:10',
+                     'TTY': '?',
+                     'UID': None,
+                     'USER': 'root',
+                     'VSZ': 195712.0,
+                     'WCHAN': None}
+    assert ps[1000] is None
+    assert [proc for proc in ps]
+
+
+def test_docs():
+    ps_alxwww = PsAlxwww(context_wrap(PS_ALXWWW_LINES))
+    ps_auxww = PsAuxww(context_wrap(PS_AUXWW_LINES))
+    ps_aux = PsAux(context_wrap(PS_AUX_LINES))
+    ps_ef = PsEf(context_wrap(PS_EF_LINES))
+    ps_auxcww = PsAuxcww(context_wrap(PS_AUXCWW_LINES))
+    ps_eo = PsEo(context_wrap(PS_EO_LINES, strip=False))
+    ps_combiner = Ps(ps_alxwww, ps_auxww, ps_aux, ps_ef, ps_auxcww, ps_eo)
+    env = {
+        'ps_combiner': ps_combiner
+    }
+    failed, total = doctest.testmod(ps, globs=env)
+    assert failed == 0