Repository: whot/uji

Issue:
Allow for specifying a "not" filter
Found this while using uji for the recent xserver CVEs: I needed a way to specify a test for actors that *don't* have a particular tag. e.g. `not rhel`.

Probably simple enough to add with `!` support.

Discussion:
bentiss: Looking good

Patch:
diff --git a/examples/example.yaml b/examples/example.yaml
index 666bcf1..9b144d6 100644
--- a/examples/example.yaml
+++ b/examples/example.yaml
@@ -36,7 +36,7 @@
 #    description: a longer description for human consumption
 #    tags: a dictionary of key: value that can be used to filter on
 #          value must not be a list/dict
-#  
+#
 # The test type is a description of a test that may produce log files.
 # Allowed subkeys:
 #    extends: inherit all from the referenced section
@@ -47,6 +47,8 @@
 #            The value must be a list, even if it's just one entry
 #            Where the filter tag is missing, this test is assumed
 #            to be run only once.
+#            If a value starts with ! it's a negative match, i.e.
+#            the tag must NOT be present in the actor.
 #    tests: a list of instruction strings describing the tests to be
 #           performed.
 #    files: a list of files to collect
@@ -80,7 +82,7 @@ file:
 #
 # Since we support 'include:' statements, you could have files with all the
 # hw you have defined in mice.yaml, keyboards.yaml, etc.
-  
+
 t450_keyboard:
   type: actor
   name: Lenovo T450s - AT Translated Keyboard
@@ -178,6 +180,15 @@ test_usb_hid:
   tests:
     - verify hid report descriptor parses with `hid-parse`
 
+# A test to run only on non-USB keyboards
+test_nonusb:
+  type: test
+  filter:
+    device: [keyboard]
+    bus: ["!USB"]  # Note: YAML requires quotes
+  tests:
+    - "bus type for this keyboard is XXX"
+
 # This is a test without a filter, so it will show up in the "Generic"
 # section but not for any specific actor. Good for things you need to
 # collect only once.
diff --git a/uji.py b/uji.py
index e5d0b6f..59bf2e4 100755
--- a/uji.py
+++ b/uji.py
@@ -736,8 +736,19 @@ class UjiNew(object):
                 for key, values in test.filters.items():
                     if key not in actor.tags:
                         break
-                    if ('__any__' not in values and
-                            actor.tags[key] not in values):
+
+                    tag = actor.tags[key]
+
+                    excluded = [v[1:] for v in values if v[0] == '!']
+                    if tag in excluded:
+                        break
+
+                    required = [v for v in values if v[0] != '!']
+                    if not required and excluded:
+                        required = ['__any__']
+
+                    if ('__any__' not in required and
+                        actor.tags[key] not in required):
                         break
                 else:
                     dup = deepcopy(test)


Test patch:
diff --git a/tests/data/basic-tree.yaml b/tests/data/basic-tree.yaml
index 6e0e83a..61f3734 100644
--- a/tests/data/basic-tree.yaml
+++ b/tests/data/basic-tree.yaml
@@ -13,16 +13,16 @@ actor2:
 # generic test
 test1:
     type: test
-    test:
-        - testcase0
+    tests:
+        - testcase1 (generic)
     logs:
-        files: [file1]
+        files: [file01-generic]
 
 # generic test
 test2:
     type: test
     logs:
-        files: [file2]
+        files: [file02-generic]
 
 # all actors but not generic
 test3:
@@ -30,22 +30,38 @@ test3:
     filter:
         actor: [__any__]
     tests:
-        - testcase1
+        - testcase3 (all actors)
 
 test4:
     type: test
     filter:
         actor: [one]
     tests:
-        - testcase2
+        - testcase4 (actor one only)
     logs:
-        files: [file3]
+        files: [file04-actor-one]
 
 test5:
     type: test
     filter:
         actor: [two]
     tests:
-        - testcase3
+        - testcase5 (actor two only)
+        - testcase5.1 (actor two only)
+        - testcase5.2 (actor two only)
     logs:
-        files: [file4]
+        files: [file05-actor-two]
+
+test6:
+    type: test
+    filter:
+        actor: ["!two"]
+    tests:
+        - testcase6 (actor one only)
+
+test7:
+    type: test
+    filter:
+        actor: ["!one", "two"]
+    tests:
+        - testcase7 (actor two only)
diff --git a/tests/test_uji.py b/tests/test_uji.py
index d5381ad..c49040a 100644
--- a/tests/test_uji.py
+++ b/tests/test_uji.py
@@ -1,5 +1,7 @@
 #!/usr/bin/env python3
 
+from typing import Optional
+
 from click.testing import CliRunner
 import pytest
 import os
@@ -16,6 +18,22 @@ def datadir():
     return Path(os.path.realpath(__file__)).parent / 'data'
 
 
+def find_in_section(markdown: str, section: str, string: str) -> Optional[str]:
+    prev_line = None
+    in_section = False
+    for line in markdown.split('\n'):
+        if prev_line is not None and prev_line == section and line == '-' * len(section):
+            in_section = True
+        elif in_section and line == '':
+            in_section = False
+        elif in_section:
+            if string in line:
+                return line
+        prev_line = line
+
+    return None
+
+
 def test_uji_example(datadir):
     args = ['new', os.fspath(Path(datadir) / 'example.yaml')]
     runner = CliRunner()
@@ -49,12 +67,39 @@ def test_uji_tree(datadir):
         assert 'actor2\n------\n' in markdown
         assert 'Generic\n-------\n' in markdown
 
-        # FIXME: check for the tests to be distributed across the actors
+        # check for the tests to be distributed across the actors
         # correctly
+        assert find_in_section(markdown, 'Generic', 'testcase1')
+        assert find_in_section(markdown, 'Generic', 'file01')
+        assert find_in_section(markdown, 'Generic', 'file02')
+
+        assert find_in_section(markdown, 'actor1', 'testcase3')
+        assert find_in_section(markdown, 'actor2', 'testcase3')
+
+        assert find_in_section(markdown, 'actor1', 'testcase4')
+        assert find_in_section(markdown, 'actor1', 'file04')
+        assert not find_in_section(markdown, 'actor2', 'testcase4')
+        assert not find_in_section(markdown, 'actor2', 'file04')
+
+        assert not find_in_section(markdown, 'actor1', 'testcase5')
+        assert not find_in_section(markdown, 'actor1', 'testcase5.1')
+        assert not find_in_section(markdown, 'actor1', 'testcase5.2')
+        assert not find_in_section(markdown, 'actor1', 'file05')
+
+        assert find_in_section(markdown, 'actor2', 'testcase5')
+        assert find_in_section(markdown, 'actor2', 'testcase5.1')
+        assert find_in_section(markdown, 'actor2', 'testcase5.2')
+        assert find_in_section(markdown, 'actor2', 'file05')
+
+        assert find_in_section(markdown, 'actor1', 'testcase6'), markdown
+        assert not find_in_section(markdown, 'actor2', 'testcase6'), markdown
+
+        assert not find_in_section(markdown, 'actor1', 'testcase7'), markdown
+        assert find_in_section(markdown, 'actor2', 'testcase7'), markdown
 
         # Check for the 'emtpy' files to be created
-        assert (Path('testdir') / 'generic' / 'test1' / 'file1').exists()
-        assert (Path('testdir') / 'generic' / 'test2' / 'file2').exists()
-        assert (Path('testdir') / 'actor1' / 'test4' / 'file3').exists()
-        assert (Path('testdir') / 'actor2' / 'test5' / 'file4').exists()
+        assert (Path('testdir') / 'generic' / 'test1' / 'file01-generic').exists()
+        assert (Path('testdir') / 'generic' / 'test2' / 'file02-generic').exists()
+        assert (Path('testdir') / 'actor1' / 'test4' / 'file04-actor-one').exists()
+        assert (Path('testdir') / 'actor2' / 'test5' / 'file05-actor-two').exists()