Repository: RedHatInsights/insights-core

Issue:
FileListing parser does not handle filenames with newlines in the name
The FileListing (and by extension ls_parser) cannot currently handle file names with newlines.  For example:

```
/etc:
total 1652
drwxr-xr-x. 102   0   0  12288 Nov  6 09:12 .
dr-xr-xr-x.  21   0   0   4096 Oct 23 10:12 ..
-rw-------.   1   0   0      0 Oct  2 13:46 .pwd.lock
-rw-r--r--    1   0   0    163 Oct  2 13:45 .updated
-rw-r--r--.   1   0   0   1059 Oct  2 13:56 chrony.conf
-rw-r--r--.   1   0   0   1100 Dec  5  2017 chrony.conf.20180210135613

-rw-r-----.   1   0 993    481 Sep 14  2017 chrony.keys
drwxr-xr-x.   2   0   0   4096 Nov  1 03:34 cifs-utils
```

The _extra_ newline will cause the parser to stop collecting files for `/etc`.


Discussion:

Resolving patch:
diff --git a/insights/core/ls_parser.py b/insights/core/ls_parser.py
--- a/insights/core/ls_parser.py
+++ b/insights/core/ls_parser.py
@@ -2,10 +2,7 @@
 This module contains logic for parsing ls output. It attempts to handle
 output when selinux is enabled or disabled and also skip "bad" lines.
 """
-import logging
-
-log = logging.getLogger(__name__)
-PERMBITS = set("-+.dlbcpsrwxtT")
+import six
 
 
 def parse_path(path):
@@ -101,7 +98,87 @@ def parse_selinux(parts):
     return result
 
 
-def parse(lines, root):
+PASS_KEYS = set(["name", "total"])
+DELAYED_KEYS = ["entries", "files", "dirs", "specials"]
+
+
+class Directory(dict):
+    def __init__(self, name, total, body):
+        data = dict.fromkeys(DELAYED_KEYS)
+        data["name"] = name
+        data["total"] = total
+        self.body = body
+        self.loaded = False
+        super(Directory, self).__init__(data)
+
+    def iteritems(self):
+        if not self.loaded:
+            self._load()
+        return six.iteritems(super(Directory, self))
+
+    def items(self):
+        if not self.loaded:
+            self._load()
+        return super(Directory, self).items()
+
+    def values(self):
+        if not self.loaded:
+            self._load()
+        return super(Directory, self).values()
+
+    def get(self, key, default=None):
+        if not self.loaded:
+            self._load()
+        return super(Directory, self).get(key, default)
+
+    def _load(self):
+        dirs = []
+        ents = {}
+        files = []
+        specials = []
+        for line in self.body:
+            parts = line.split(None, 4)
+            perms = parts[0]
+            typ = perms[0]
+            entry = {
+                "type": typ,
+                "perms": perms[1:]
+            }
+            if parts[1][0].isdigit():
+                rest = parse_non_selinux(parts[1:])
+            else:
+                rest = parse_selinux(parts[1:])
+
+            # Update our entry and put it into the correct buckets
+            # based on its type.
+            entry.update(rest)
+            entry["raw_entry"] = line
+            entry["dir"] = self["name"]
+            nm = entry["name"]
+            ents[nm] = entry
+            if typ not in "bcd":
+                files.append(nm)
+            elif typ == "d":
+                dirs.append(nm)
+            elif typ in "bc":
+                specials.append(nm)
+
+        self.update({"entries": ents,
+                     "files": files,
+                     "dirs": dirs,
+                     "specials": specials})
+
+        self.loaded = True
+        del self.body
+
+    def __getitem__(self, key):
+        if self.loaded or key in PASS_KEYS:
+            return super(Directory, self).__getitem__(key)
+        self._load()
+        return super(Directory, self).__getitem__(key)
+
+
+def parse(lines, root=None):
     """
     Parses a list of lines from ls into dictionaries representing their
     components.
@@ -115,117 +192,33 @@ def parse(lines, root):
         A dictionary representing the ls output. It's keyed by the path
         containing each ls stanza.
     """
-    lines = iter(lines)
-    results = {}
-    # we process until StopIteration exception is thrown by the line iterator.
-    while True:
-        try:
-            # get a line and skip blanks
-            line = next(lines)
-            while not(line):
-                line = next(lines)
-            try:
-                # an ls stanza begins with an optional directory name followed
-                # by a total number of entries in the directory and then the
-                # list of entries. Stanzas are separated by blank lines.
-                name = None
+    doc = {}
+    entries = []
+    name = None
+    total = None
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        if line and line[0] == "/" and line[-1] == ":":
+            if name is None:
+                name = line[:-1]
+                if entries:
+                    d = Directory(name, total or len(entries), entries)
+                    doc[root] = d
+                    total = None
+                    entries = []
+            else:
+                d = Directory(name, total or len(entries), entries)
+                doc[name or root] = d
                 total = None
-                if line.endswith(":"):
-                    name = line[:-1]
-                    line = next(lines).strip()
-                if line.startswith("total"):
-                    total = int(line.split()[1])
-                    line = next(lines).strip()
-
-                # If we didn't get a name or a total, it's not a valid stanza
-                # start, so just start over.
-                if not name and not total:
-                    continue
-
-                # handle unnamed ls stanzas
-                name = name or root
-
-                # keep track of directories, files, special entries like block
-                # and character devices, and a master list of all entries
-                # regardless of kind.
-                dirs = []
-                ents = {}
-                files = []
-                specials = []
-
-                # make sure we have a good line and its first character is one
-                # of the valid permission bits.
-                while line and line[0] in PERMBITS:
-                    # break the line into initial parts
-                    parts = line.split(None, 4)
-
-                    # Get the permissions and validate they're all valid bits.
-                    # Otherwise, our single character check in the loop was
-                    # fooled, and we should skip the line as invalid.
-                    perms = parts[0]
-                    if len(perms) < 10:
-                        line = next(lines).strip()
-                        continue
-
-                    # First character of the perms is the type of entry:
-                    # directory, regular file, etc. The rest are the actual
-                    # permission bits.
-                    typ = perms[0]
-                    entry = {
-                        "type": typ,
-                        "perms": perms[1:]
-                    }
-
-                    # If the first character of the second part of the line is
-                    # a digit, then it's a link count and indicates we're not
-                    # dealing with selinux. Otherwise, it's an owner, and we
-                    # are dealing with selinux.
-                    if parts[1][0].isdigit():
-                        rest = parse_non_selinux(parts[1:])
-                    else:
-                        rest = parse_selinux(parts[1:])
-
-                    # Update our entry and put it into the correct buckets
-                    # based on its type.
-                    entry.update(rest)
-                    entry["raw_entry"] = line
-                    entry["dir"] = name
-                    nm = entry["name"]
-                    ents[nm] = entry
-                    if typ not in "bcd":
-                        files.append(nm)
-                    elif typ == "d":
-                        dirs.append(nm)
-                    elif typ in "bc":
-                        specials.append(nm)
-                    try:
-                        line = next(lines).strip()
-                    except StopIteration:
-                        break
-
-                # We're either at the end of input or a blank line, so we wrap
-                # up this stanza and save it.
-                if total is None:
-                    total = len(ents)
-
-                result = {
-                    "name": name,
-                    "total": total,
-                    "entries": ents,
-                    "files": files,
-                    "dirs": dirs,
-                    "specials": specials
-                }
-                results[result["name"]] = result
-
-            # If we're at the end of input, blow all the way out.
-            except StopIteration:
-                raise
-
-            # Something exploded. Just log it and try to move on.
-            except:
-                line = next(lines)
-                log.info("Failed to parse: %s" % line)
-        except StopIteration:
-            break
-    return results
+                entries = []
+                name = line[:-1]
+            continue
+        if line.startswith("total"):
+            total = int(line.split(None, 1)[1])
+            continue
+        entries.append(line)
+    name = name or root
+    doc[name] = Directory(name, total or len(entries), entries)
+    return doc


Test patch:
diff --git a/insights/tests/test_ls_parser.py b/insights/tests/test_ls_parser.py
--- a/insights/tests/test_ls_parser.py
+++ b/insights/tests/test_ls_parser.py
@@ -1,4 +1,5 @@
 # -*- coding: UTF-8 -*-
+import six
 from insights.core.ls_parser import parse
 
 
@@ -33,6 +34,39 @@
 lrwxrwxrwx.  1 0 0   15 Jul  6 23:32 S97rhnsd -> ../init.d/rhnsd
 """
 
+MULTIPLE_DIRECTORIES_WITH_BREAK = """
+/etc:
+total 1652
+drwxr-xr-x. 102   0   0  12288 Nov  6 09:12 .
+dr-xr-xr-x.  21   0   0   4096 Oct 23 10:12 ..
+-rw-------.   1   0   0      0 Oct  2 13:46 .pwd.lock
+-rw-r--r--    1   0   0    163 Oct  2 13:45 .updated
+-rw-r--r--.   1   0   0   1059 Oct  2 13:56 chrony.conf
+-rw-r--r--.   1   0   0   1100 Dec  5  2017 chrony.conf.20180210135613
+
+-rw-r-----.   1   0 993    481 Sep 14  2017 chrony.keys
+drwxr-xr-x.   2   0   0   4096 Nov  1 03:34 cifs-utils
+
+/etc/sysconfig:
+total 96
+drwxr-xr-x.  7 0 0 4096 Jul  6 23:41 .
+drwxr-xr-x. 77 0 0 8192 Jul 13 03:55 ..
+drwxr-xr-x.  2 0 0   41 Jul  6 23:32 cbq
+drwxr-xr-x.  2 0 0    6 Sep 16  2015 console
+-rw-------.  1 0 0 1390 Mar  4  2014 ebtables-config
+-rw-r--r--.  1 0 0   72 Sep 15  2015 firewalld
+lrwxrwxrwx.  1 0 0   17 Jul  6 23:32 grub -> /etc/default/grub
+
+/etc/rc.d/rc3.d:
+total 4
+drwxr-xr-x.  2 0 0   58 Jul  6 23:32 .
+drwxr-xr-x. 10 0 0 4096 Sep 16  2015 ..
+lrwxrwxrwx.  1 0 0   20 Jul  6 23:32 K50netconsole -> ../init.d/netconsole
+lrwxrwxrwx.  1 0 0   17 Jul  6 23:32 S10network -> ../init.d/network
+lrwxrwxrwx.  1 0 0   15 Jul  6 23:32 S97rhnsd -> ../init.d/rhnsd
+"""
+
+
 COMPLICATED_FILES = """
 /tmp:
 total 16
@@ -163,7 +197,7 @@ def test_parse_single_directory():
 
 def test_parse_multiple_directories():
     results = parse(MULTIPLE_DIRECTORIES.splitlines(), None)
-    assert len(results) == 2
+    assert len(results) == 2, len(results)
     assert results["/etc/sysconfig"]["name"] == "/etc/sysconfig"
     assert results["/etc/sysconfig"]["total"] == 96
     assert results["/etc/rc.d/rc3.d"]["name"] == "/etc/rc.d/rc3.d"
@@ -180,6 +214,28 @@ def test_parse_multiple_directories():
     assert res["dir"] == "/etc/sysconfig"
 
 
+def test_parse_multiple_directories_with_break():
+    results = parse(MULTIPLE_DIRECTORIES_WITH_BREAK.splitlines(), None)
+    assert len(results) == 3, len(results)
+    assert len(results.values()) == 3
+    assert len(results.items()) == 3
+    assert len(list(six.iteritems(results))) == 3
+    assert results["/etc"]["name"] == "/etc"
+    assert results["/etc"]["total"] == 1652
+    assert results["/etc/rc.d/rc3.d"]["name"] == "/etc/rc.d/rc3.d"
+    assert results["/etc/rc.d/rc3.d"]["total"] == 4
+
+    res = results["/etc"]["entries"]["chrony.conf.20180210135613"]
+    assert res["type"] == "-"
+    assert res["links"] == 1
+    assert res["owner"] == "0"
+    assert res["group"] == "0"
+    assert res["size"] == 1100
+    assert res["date"] == "Dec  5  2017"
+    assert res["name"] == "chrony.conf.20180210135613"
+    assert res["dir"] == "/etc"
+
+
 def test_complicated_files():
     results = parse(COMPLICATED_FILES.splitlines(), "/tmp")
     assert len(results) == 1