Repository: RedHatInsights/insights-core

Issue:
Extend insights.core.filters.add_filter() functionality
Extend the functionality of `insights.core.filters.add_filter()` to allow developers to filters underlying datasources by adding filter(s) to a component like parser or combiner. When a filter is added to a non-datasource component, the function logic needs to locate filterable datasources in the dependency graph and correctly filter those.


Discussion:

Resolving patch:
diff --git a/insights/core/filters.py b/insights/core/filters.py
--- a/insights/core/filters.py
+++ b/insights/core/filters.py
@@ -1,10 +1,15 @@
 """
-The filters module allows developers to apply filters to datasources. A filter
-is a simple string, and it matches if it is contained anywhere within a line.
+The filters module allows developers to apply filters to datasources,
+by adding them directly or through dependent components like parsers
+and combiners. A filter is a simple string, and it matches if it is contained
+anywhere within a line.
 
 If a datasource has filters defined, it will return only lines matching at
 least one of them. If a datasource has no filters, it will return all lines.
 
+Filters can be added to components like parsers and combiners, to apply consistent
+filtering to multiple underlying datasources that are configured as filterable.
+
 Filters aren't applicable to "raw" datasources, which are created with
 ``kind=RawFileProvider`` and have ``RegistryPoint`` instances with ``raw=True``.
 
@@ -46,37 +51,49 @@
 ENABLED = parse_bool(os.environ.get("INSIGHTS_FILTERS_ENABLED"), default=True)
 
 
-def add_filter(ds, patterns):
+def add_filter(component, patterns):
     """
-    Add a filter or list of filters to a datasource. A filter is a simple
-    string, and it matches if it is contained anywhere within a line.
+    Add a filter or list of filters to a component. When the component is
+    a datasource, the filter will be directly added to that datasouce.
+    In cases when the component is a parser or combiner, the filter will be
+    added to underlying filterable datasources by traversing dependency graph.
+    A filter is a simple string, and it matches if it is contained anywhere
+    within a line.
 
     Args:
-       ds (@datasource component): The datasource to filter
+       component (component): The component to filter, can be datasource,
+            parser or combiner.
        patterns (str, [str]): A string, list of strings, or set of strings to
             add to the datasource's filters.
     """
-    if not plugins.is_datasource(ds):
-        raise Exception("Filters are applicable only to datasources.")
-
-    delegate = dr.get_delegate(ds)
+    def inner(component, patterns):
+        if component in _CACHE:
+            del _CACHE[component]
+        if isinstance(patterns, six.string_types):
+            FILTERS[component].add(patterns)
+        elif isinstance(patterns, list):
+            FILTERS[component] |= set(patterns)
+        elif isinstance(patterns, set):
+            FILTERS[component] |= patterns
+        else:
+            raise TypeError("patterns must be string, list, or set.")
+
+    if not plugins.is_datasource(component):
+        for dep in dr.run_order(dr.get_dependency_graph(component)):
+            if plugins.is_datasource(dep):
+                d = dr.get_delegate(dep)
+                if d.filterable:
+                    inner(dep, patterns)
+    else:
+        delegate = dr.get_delegate(component)
 
-    if delegate.raw:
-        raise Exception("Filters aren't applicable to raw datasources.")
+        if delegate.raw:
+            raise Exception("Filters aren't applicable to raw datasources.")
 
-    if not delegate.filterable:
-        raise Exception("Filters aren't applicable to %s." % dr.get_name(ds))
+        if not delegate.filterable:
+            raise Exception("Filters aren't applicable to %s." % dr.get_name(component))
 
-    if ds in _CACHE:
-        del _CACHE[ds]
-    if isinstance(patterns, six.string_types):
-        FILTERS[ds].add(patterns)
-    elif isinstance(patterns, list):
-        FILTERS[ds] |= set(patterns)
-    elif isinstance(patterns, set):
-        FILTERS[ds] |= patterns
-    else:
-        raise TypeError("patterns must be string, list, or set.")
+        inner(component, patterns)
 
 
 _add_filter = add_filter
diff --git a/insights/plugins/ps_rule_fakes.py b/insights/plugins/ps_rule_fakes.py
new file mode 100644
--- /dev/null
+++ b/insights/plugins/ps_rule_fakes.py
@@ -0,0 +1,25 @@
+from insights import rule, make_pass
+from insights.core.filters import add_filter
+from insights.parsers.ps import PsAux, PsAuxww, PsAlxwww
+from insights.specs import Specs
+
+
+@rule(PsAux)
+def psaux_no_filter(ps_aux):
+    return make_pass("FAKE RESULT")
+
+
+add_filter(Specs.ps_auxww, "fake-filter")
+
+
+@rule(PsAuxww)
+def psauxww_ds_filter(ps_auxww):
+    return make_pass("FAKE RESULT")
+
+
+add_filter(PsAlxwww, "fake-filter")
+
+
+@rule(PsAlxwww)
+def psalxwww_parser_filter(ps_alxwww):
+    return make_pass("FAKE RESULT")


Test patch:
diff --git a/insights/tests/__init__.py b/insights/tests/__init__.py
--- a/insights/tests/__init__.py
+++ b/insights/tests/__init__.py
@@ -32,10 +32,10 @@
 
 def _intercept_add_filter(func):
     @wraps(func)
-    def inner(ds, pattern):
-        ret = add_filter(ds, pattern)
+    def inner(component, pattern):
+        ret = add_filter(component, pattern)
         calling_module = inspect.stack()[1][0].f_globals.get("__name__")
-        ADDED_FILTERS[calling_module].add(ds)
+        ADDED_FILTERS[calling_module] |= set(r for r in _get_registry_points(component) if r.filterable)
         return ret
     return inner
 
@@ -57,7 +57,13 @@ def inner(ds, pattern):
 
 
 def _get_registry_points(component):
+    """
+    Get underlying registry points for a component. The return set
+    will include the passed-in component if it is also a registry point.
+    """
     results = set()
+    if isinstance(component, RegistryPoint):
+        results.add(component)
     for c in dr.walk_tree(component):
         try:
             if isinstance(c, RegistryPoint):
diff --git a/insights/tests/test_filters.py b/insights/tests/test_filters.py
--- a/insights/tests/test_filters.py
+++ b/insights/tests/test_filters.py
@@ -1,9 +1,12 @@
 from collections import defaultdict
 from insights.core import filters
 
+from insights.parsers.ps import PsAux, PsAuxcww
 from insights.specs import Specs
 from insights.specs.default import DefaultSpecs
 
+import pytest
+
 
 def setup_function(func):
     if func is test_get_filter:
@@ -28,6 +31,12 @@ def teardown_function(func):
     if func is test_filter_dumps_loads:
         del filters.FILTERS[Specs.ps_aux]
 
+    if func is test_add_filter_to_parser:
+        del filters.FILTERS[Specs.ps_aux]
+
+    if func is test_add_filter_to_parser_patterns_list:
+        del filters.FILTERS[Specs.ps_aux]
+
 
 def test_filter_dumps_loads():
     r = filters.dumps()
@@ -56,3 +65,46 @@ def test_get_filter_registry_point():
     f = filters.get_filters(Specs.ps_aux)
     assert "COMMAND" in f
     assert "MEM" not in f
+
+
+def test_add_filter_to_parser():
+    filter_string = "bash"
+    filters.add_filter(PsAux, filter_string)
+
+    spec_filters = filters.get_filters(Specs.ps_aux)
+    assert filter_string in spec_filters
+
+    parser_filters = filters.get_filters(PsAux)
+    assert not parser_filters
+
+
+def test_add_filter_to_parser_patterns_list():
+    filters_list = ["bash", "systemd", "Network"]
+    filters.add_filter(PsAux, filters_list)
+
+    spec_filters = filters.get_filters(Specs.ps_aux)
+    assert all(f in spec_filters for f in filters_list)
+
+    parser_filters = filters.get_filters(PsAux)
+    assert not parser_filters
+
+
+def test_add_filter_to_parser_non_filterable():
+    filter_string = "bash"
+    filters.add_filter(PsAuxcww, filter_string)
+
+    spec_filters = filters.get_filters(Specs.ps_auxcww)
+    assert not spec_filters
+
+    parser_filters = filters.get_filters(PsAuxcww)
+    assert not parser_filters
+
+
+def test_add_filter_exception_not_filterable():
+    with pytest.raises(Exception):
+        filters.add_filter(Specs.ps_auxcww, "bash")
+
+
+def test_add_filter_exception_raw():
+    with pytest.raises(Exception):
+        filters.add_filter(Specs.metadata_json, "[]")
diff --git a/insights/tests/test_integration_support.py b/insights/tests/test_integration_support.py
new file mode 100644
--- /dev/null
+++ b/insights/tests/test_integration_support.py
@@ -0,0 +1,39 @@
+from insights.plugins.ps_rule_fakes import psaux_no_filter, psauxww_ds_filter, psalxwww_parser_filter
+from insights.specs import Specs
+from . import InputData, run_test
+
+import pytest
+
+
+def test_run_test_missing_filters_exception():
+    """
+    The rule underlying datasource requires a filter,
+    an exception should be raised because filter was not
+    added in the rule module.
+    """
+    input_data = InputData("fake_input")
+    input_data.add(Specs.ps_aux, "FAKE_CONTENT")
+    with pytest.raises(Exception):
+        run_test(psaux_no_filter, input_data, None)
+
+
+def test_run_test_no_missing_filters_using_datasource():
+    """
+    Required filter was added directly to the datasouce,
+    ``run_test`` should complete without any exceptions.
+    """
+    input_data = InputData("fake_input")
+    input_data.add(Specs.ps_auxww, "FAKE_CONTENT")
+    result = run_test(psauxww_ds_filter, input_data, None)
+    assert result
+
+
+def test_run_test_no_missing_filters_using_parser():
+    """
+    Required filter was added to using the parser,
+    ``run_test`` should complete without any exceptions.
+    """
+    input_data = InputData("fake_input")
+    input_data.add(Specs.ps_alxwww, "FAKE_CONTENT")
+    result = run_test(psalxwww_parser_filter, input_data, None)
+    assert result