Repository: RedHatInsights/insights-core

Issue:
parsr.query performance issues
The parsr.query data structure uses ~1GB of memory for ~3 million nodes. The dict and list representation has about a third the footprint.


Discussion:

Resolving patch:
diff --git a/insights/parsr/query/__init__.py b/insights/parsr/query/__init__.py
--- a/insights/parsr/query/__init__.py
+++ b/insights/parsr/query/__init__.py
@@ -22,9 +22,17 @@
 """
 import operator
 import re
+import six
+import sys
 from collections import defaultdict
-from itertools import chain
-from insights.parsr.query.boolean import All, Any, Boolean, Not, pred, pred2, TRUE
+from itertools import chain, count
+from insights.parsr.query.boolean import All, Any, Boolean, Not, pred, pred2  # noqa
+
+# intern was a builtin until it moved to sys in python3
+try:
+    intern = sys.intern
+except:
+    pass
 
 
 class Entry(object):
@@ -35,9 +43,9 @@ class Entry(object):
     __slots__ = ("_name", "attrs", "children", "parent", "lineno", "src")
 
     def __init__(self, name=None, attrs=None, children=None, lineno=None, src=None):
-        self._name = name
-        self.attrs = attrs or []
-        self.children = children or []
+        self._name = intern(six.ensure_str(name)) if name is not None else name
+        self.attrs = attrs if isinstance(attrs, (list, tuple)) else tuple()
+        self.children = children if isinstance(children, (list, tuple)) else []
         self.parent = None
         self.lineno = lineno
         self.src = src
@@ -67,7 +75,7 @@ def get_keys(self):
         """
         Returns the unique names of all the children as a list.
         """
-        return sorted(set(c.name for c in self.children))
+        return sorted(set(c._name for c in self.children if c._name))
 
     def __dir__(self):
         """
@@ -92,7 +100,7 @@ def line(self):
         """
         Returns the original first line of text that generated the ``Entry``.
         """
-        if self.src is not None:
+        if self.src is not None and self.lineno is not None:
             return self.src.content[self.lineno - 1]
 
     @property
@@ -136,10 +144,10 @@ def upto(self, query):
         """
         Go up from the current node to the first node that matches query.
         """
-        pred = _desugar(query)
+        predicate = _desugar(query)
         parent = self.parent
         while parent is not None:
-            if pred.test(parent):
+            if predicate(parent):
                 return parent
             parent = parent.parent
 
@@ -182,14 +190,19 @@ def where(self, name, value=None):
         ['2019-08-04T23:17:08Z', '2019-08-04T23:32:14Z']
         """
         if isinstance(name, _EntryQuery):
-            query = name
+            query = name.to_pyfunc()
         elif isinstance(name, Boolean):
-            query = child_query(name, value)
+            query = child_query(name, value).to_pyfunc()
         elif callable(name):
-            query = SimpleQuery(pred(name))
+            def predicate(e):
+                try:
+                    return name(e)
+                except:
+                    return False
+            query = predicate
         else:
-            query = child_query(name, value)
-        return Result(children=self.children if query.test(self) else [])
+            query = child_query(name, value).to_pyfunc()
+        return Result(children=self.children if query(self) else [])
 
     @property
     def section(self):
@@ -225,7 +238,7 @@ def __getitem__(self, query):
         if isinstance(query, (int, slice)):
             return self.children[query]
         query = _desugar(query)
-        return Result(children=[c for c in self.children if query.test(c)])
+        return Result(children=[c for c in self.children if query(c)])
 
     def __bool__(self):
         return bool(self._name or self.attrs or self.children)
@@ -416,7 +429,7 @@ def nth(self, n):
             tmp[c.parent].append(c)
 
         results = []
-        for p, v in tmp.items():
+        for _, v in tmp.items():
             try:
                 r = v[n]
                 if isinstance(r, list):
@@ -451,17 +464,22 @@ def where(self, name, value=None):
         ['2019-08-04T23:17:08Z', '2019-08-04T23:32:14Z']
         """
         if isinstance(name, _EntryQuery):
-            query = name
+            query = name.to_pyfunc()
         elif isinstance(name, Boolean):
-            query = child_query(name, value)
+            query = child_query(name, value).to_pyfunc()
         elif callable(name):
-            query = SimpleQuery(pred(name))
+            def predicate(e):
+                try:
+                    return name(e)
+                except:
+                    return False
+            query = predicate
         else:
-            query = child_query(name, value)
+            query = child_query(name, value).to_pyfunc()
         results = []
         seen = set()
         for c in self.children:
-            if c not in seen and query.test(c):
+            if c not in seen and query(c):
                 results.append(c)
         return Result(children=results)
 
@@ -469,7 +487,7 @@ def __getitem__(self, query):
         if isinstance(query, (int, slice)):
             return self.children[query]
         query = _desugar(query)
-        return Result(children=[c for c in self.grandchildren if query.test(c)])
+        return Result(children=[c for c in self.grandchildren if query(c)])
 
 
 class _EntryQuery(object):
@@ -485,6 +503,38 @@ def __or__(self, other):
     def __invert__(self):
         return _NotEntryQuery(self)
 
+    def to_pyfunc(self):
+        ver = sys.version_info
+        if ver[0] == 2 and ver[1] == 6:
+            return self.test
+
+        env = {}
+        ids = count()
+
+        def expr(b):
+            if isinstance(b, _AllEntryQuery):
+                return "(" + " and ".join(expr(p) for p in b.exprs) + ")"
+            elif isinstance(b, _AnyEntryQuery):
+                return "(" + " or ".join(expr(p) for p in b.exprs) + ")"
+            elif isinstance(b, _NotEntryQuery):
+                return "(" + "not " + expr(b.query) + ")"
+            else:
+                num = next(ids)
+                func = "func_{num}".format(num=num)
+                env[func] = b.test
+                return func + "(value)"
+
+        func = """
+def predicate(value):
+    try:
+        return {body}
+    except Exception as ex:
+        return False
+        """.format(body=expr(self))
+
+        six.exec_(func, env, env)
+        return env["predicate"]
+
 
 class _AllEntryQuery(_EntryQuery, All):
     pass
@@ -498,31 +548,20 @@ class _NotEntryQuery(_EntryQuery, Not):
     pass
 
 
-class NameQuery(_EntryQuery):
-    """
-    A query against the name of an :py:class:`Entry`.
-    """
-    def __init__(self, expr):
-        self.expr = expr
-
-    def test(self, n):
-        return self.expr.test(n.name)
-
-
 class _AllAttrQuery(_EntryQuery):
     def __init__(self, expr):
         self.expr = expr
 
-    def test(self, n):
-        return all(self.expr.test(a) for a in n.attrs)
+    def test(self, e):
+        return all(self.expr(a) for a in e.attrs)
 
 
 class _AnyAttrQuery(_EntryQuery):
     def __init__(self, expr):
         self.expr = expr
 
-    def test(self, n):
-        return any(self.expr.test(a) for a in n.attrs)
+    def test(self, e):
+        return any(self.expr(a) for a in e.attrs)
 
 
 def all_(expr):
@@ -541,36 +580,21 @@ def any_(expr):
     return _AnyAttrQuery(_desugar_attr(expr))
 
 
-class SimpleQuery(_EntryQuery):
-    """
-    Automatically used in ``Entry.where`` or ``Result.where``. ``SimpleQuery``
-    wraps a function or a lambda that will be passed each ``Entry`` of the
-    current result. The passed function should return ``True`` or ``False``.
-    """
-    def __init__(self, expr):
-        if not isinstance(expr, Boolean):
-            expr = pred(expr)
-        self.expr = expr
-
-    def test(self, node):
-        return self.expr.test(node)
-
-
 class ChildQuery(_EntryQuery):
     """
-    Returns True if any child node passes the query.
+    Returns True if any child entry passes the query.
     """
     def __init__(self, expr):
         self.expr = expr
 
-    def test(self, node):
-        return any(self.expr.test(n) for n in node.children)
+    def test(self, e):
+        return any(self.expr(n) for n in e.children)
 
 
 def child_query(name, value=None):
     """
-    Converts a query into a ChildQuery that works on all child nodes at once
-    to determine if the current node is accepted.
+    Converts a query into a ChildQuery that works on all child entries at once
+    to determine if the current entry is accepted.
     """
     q = _desugar((name, value) if value is not None else name)
     return ChildQuery(q)
@@ -581,22 +605,31 @@ def child_query(name, value=None):
 
 def _desugar_name(q):
     if q is None:
-        return NameQuery(TRUE)
-    if isinstance(q, NameQuery):
-        return q
+        return lambda _: True
     if isinstance(q, Boolean):
-        return NameQuery(q)
+        f = q.to_pyfunc()
+        return lambda e: f(e._name)
     if callable(q):
-        return NameQuery(pred(q))
-    return NameQuery(eq(q))
+        def predicate(e):
+            try:
+                return q(e._name)
+            except:
+                return False
+        return predicate
+    return lambda e: e._name == q
 
 
 def _desugar_attr(q):
     if isinstance(q, Boolean):
-        return q
+        return q.to_pyfunc()
     if callable(q):
-        return pred(q)
-    return eq(q)
+        def predicate(v):
+            try:
+                return q(v)
+            except:
+                return False
+        return predicate
+    return lambda v: v == q
 
 
 def _desugar_attrs(q):
@@ -606,20 +639,23 @@ def _desugar_attrs(q):
         q = q[0]
         return q if isinstance(q, _EntryQuery) else _AnyAttrQuery(_desugar_attr(q))
     else:
+        # conf[name, q0, q1] means "name and (q0 or q1 for any attribute)"
         attr_queries = [_desugar_attr(a) for a in q]
-        return _AnyAttrQuery(Any(*attr_queries))
+        return _AnyAttrQuery(lambda v: any(p(v) for p in attr_queries))
 
 
 def _desugar(q):
     if isinstance(q, _EntryQuery):
-        return q
+        return q.to_pyfunc()
     if isinstance(q, tuple):
-        q = list(q)
         name_query = _desugar_name(q[0])
         attrs_query = _desugar_attrs(q[1:])
         if attrs_query:
-            return All(name_query, attrs_query)
+            aq = attrs_query.to_pyfunc()
+            return lambda e: name_query(e) and aq(e)
         return name_query
+    if q is None:
+        return lambda _: True
     return _desugar_name(q)
 
 
@@ -628,9 +664,9 @@ def _flatten(nodes):
     Flatten the config tree into a list of nodes.
     """
     def inner(n):
-        res = [n]
-        res.extend(chain.from_iterable(inner(c) for c in n.children))
-        return res
+        yield n
+        for i in chain.from_iterable(inner(c) for c in n.children):
+            yield i
     return list(chain.from_iterable(inner(n) for n in nodes))
 
 
@@ -647,11 +683,13 @@ def compile_queries(*queries):
     are `or'd` together and that result is `anded` with the name query. Any
     query that raises an exception is treated as ``False``.
     """
+    queries = [_desugar(q) for q in queries]
+
     def match(qs, nodes):
-        q = _desugar(qs[0])
-        res = [n for n in nodes if q.test(n)]
+        q = qs[0]
         qs = qs[1:]
-        if qs:
+        res = [n for n in nodes if q(n)]
+        if qs and res:
             gc = list(chain.from_iterable(n.children for n in res))
             return match(qs, gc)
         return res
@@ -701,12 +739,12 @@ def inner(d):
                     if isinstance(res[0], Entry):
                         result.extend(res)
                     else:
-                        result.append(Entry(name=k, attrs=res))
+                        result.append(Entry(name=k, attrs=tuple(res)))
                 else:
-                    result.append(Entry(name=k, attrs=[]))
+                    result.append(Entry(name=k))
             else:
-                result.append(Entry(name=k, attrs=[v]))
-        return result
+                result.append(Entry(name=k, attrs=(v,)))
+        return tuple(result)
     return Entry(children=inner(orig))
 
 
diff --git a/insights/parsr/query/boolean.py b/insights/parsr/query/boolean.py
--- a/insights/parsr/query/boolean.py
+++ b/insights/parsr/query/boolean.py
@@ -39,6 +39,9 @@ def is_positive(n):
         gt_five_and_lt_10 = gt(5) & lt(10)
 
 """
+from itertools import count
+import six
+import sys
 
 
 class Boolean(object):
@@ -57,6 +60,51 @@ def test(self, value):
     def __call__(self, value):
         return self.test(value)
 
+    def to_pyfunc(self):
+        ver = sys.version_info
+        if ver[0] == 2 and ver[1] == 6:
+            return self.test
+
+        env = {}
+        ids = count()
+
+        def expr(b):
+            if b is TRUE:
+                return " True "
+            elif b is FALSE:
+                return " False "
+            elif isinstance(b, All):
+                return "(" + " and ".join(expr(p) for p in b.exprs) + ")"
+            elif isinstance(b, Any):
+                return "(" + " or ".join(expr(p) for p in b.exprs) + ")"
+            elif isinstance(b, Not):
+                return "(" + "not " + expr(b.query) + ")"
+            elif isinstance(b, Predicate):
+                num = next(ids)
+
+                func = "func_{num}".format(num=num)
+                args = "args_{num}".format(num=num)
+
+                env[func] = b.func
+                env[args] = b.args
+
+                if isinstance(b, CaselessPredicate):
+                    return func + "(value.lower(), " + "*" + args + ")"
+                return func + "(value, " + "*" + args + ")"
+            else:
+                raise Exception(b)
+
+        func = """
+def predicate(value):
+    try:
+        return {body}
+    except Exception as ex:
+        return False
+        """.format(body=expr(self))
+
+        six.exec_(func, env, env)
+        return env["predicate"]
+
 
 class TRUE(Boolean):
     pass


Test patch: