Repository: mesonbuild/meson

Issue:
mesontest: tests should be namespaced by subproject
When running `mesontest` in a top project, it would be good if tests were automatically namespaced by subproject. This would make things clearer for users as they would know right away which component is to blame if a test is failing (without having to dig in logs).
It would also prevent naming clashes. At the moment, if two subproject have tests with the same name, running `mesontest $TEST-NAME` will run all of these and it's impossible to pick the one we actually care about.

We are hitting those two issues in GStreamer: https://bugzilla.gnome.org/show_bug.cgi?id=776718




Discussion:
Actually when running all test, they are displayed with the subproject name:

```
101/290 gstreamer / libs/gstlibscpp             OK       0.03 s
102/290 gst-plugins-base / gst/typefindfunctions  OK       0.10 s
```

So what we need here is:
-  Namespace them when running `mesontest --list` as well
-  Allow to pass the full name to `mesontest` to run a specific test
Something like this is ongoing in #1252.

Resolving patch:
diff --git a/mesonbuild/interpreter.py b/mesonbuild/interpreter.py
--- a/mesonbuild/interpreter.py
+++ b/mesonbuild/interpreter.py
@@ -2080,14 +2080,14 @@ def add_test(self, node, args, kwargs, is_base_test):
             workdir = None
         if not isinstance(timeout, int):
             raise InterpreterException('Timeout must be an integer.')
-        suite = mesonlib.stringlistify(kwargs.get('suite', ''))
-        if self.is_subproject():
-            newsuite = []
-            for s in suite:
-                if len(s) > 0:
-                    s = '.' + s
-                newsuite.append(self.subproject.replace(' ', '_').replace('.', '_') + s)
-            suite = newsuite
+        suite = []
+        for s in mesonlib.stringlistify(kwargs.get('suite', '')):
+            if len(s) > 0:
+                s = ':' + s
+            if self.is_subproject():
+                suite.append(self.subproject.replace(' ', '_').replace(':', '_') + s)
+            else:
+                suite.append(self.build.project_name.replace(' ', '_').replace(':', '_') + s)
         t = Test(args[0], suite, args[1].held_object, par, cmd_args, env, should_fail, timeout, workdir)
         if is_base_test:
             self.build.tests.append(t)
diff --git a/mesontest.py b/mesontest.py
--- a/mesontest.py
+++ b/mesontest.py
@@ -64,8 +64,10 @@ def determine_worker_count():
                     help='wrapper to run tests with (e.g. Valgrind)')
 parser.add_argument('-C', default='.', dest='wd',
                     help='directory to cd into before running')
-parser.add_argument('--suite', default=None, dest='suite',
+parser.add_argument('--suite', default=[], dest='include_suites', action='append', metavar='SUITE',
                     help='Only run tests belonging to the given suite.')
+parser.add_argument('--no-suite', default=[], dest='exclude_suites', action='append', metavar='SUITE',
+                    help='Do not run tests belonging to the given suite.')
 parser.add_argument('--no-stdsplit', default=True, dest='split', action='store_false',
                     help='Do not split stderr and stdout in test logs.')
 parser.add_argument('--print-errorlogs', default=False, action='store_true',
@@ -155,10 +157,12 @@ def __init__(self, options):
         self.timeout_count = 0
         self.is_run = False
         self.cant_rebuild = False
+        self.tests = None
+        self.suites = None
         if self.options.benchmark:
-            self.datafile = os.path.join(options.wd, 'meson-private/meson_benchmark_setup.dat')
+            self.load_datafile(os.path.join(options.wd, 'meson-private/meson_benchmark_setup.dat'))
         else:
-            self.datafile = os.path.join(options.wd, 'meson-private/meson_test_setup.dat')
+            self.load_datafile(os.path.join(options.wd, 'meson-private/meson_test_setup.dat'))
 
     def rebuild_all(self):
         if not os.path.isfile(os.path.join(self.options.wd, 'build.ninja')):
@@ -325,16 +329,56 @@ def doit(self):
         self.run_tests(tests)
         return self.fail_count
 
-    def get_tests(self):
+    def split_suite_string(suite):
+        if ':' in suite:
+            return suite.split(':', 1)
+        else:
+            return (suite, "")
+
+    def test_in_suites(test, suites):
+        for suite in suites:
+            (prj_match, st_match) = TestHarness.split_suite_string(suite)
+            for prjst in test.suite:
+                (prj, st) = TestHarness.split_suite_string(prjst)
+                if prj_match and prj != prj_match:
+                    continue
+                if st_match and st != st_match:
+                    continue
+                return True
+        return False
+
+    def test_suitable(self, test):
+        return (len(self.options.include_suites) == 0 or TestHarness.test_in_suites(test, self.options.include_suites)) \
+            and not TestHarness.test_in_suites(test, self.options.exclude_suites)
+
+    def load_suites(self):
+        ss = set()
+        for t in self.tests:
+            for s in t.suite:
+                ss.add(s)
+        self.suites = list(ss)
+
+    def load_tests(self):
         with open(self.datafile, 'rb') as f:
-            tests = pickle.load(f)
+            self.tests = pickle.load(f)
 
-        if not tests:
+    def load_datafile(self, datafile):
+        self.datafile = datafile
+        self.load_tests()
+        self.load_suites()
+
+    def get_tests(self):
+        if not self.tests:
             print('No tests defined.')
             return []
 
-        if self.options.suite:
-            tests = [t for t in tests if self.options.suite in t.suite]
+        if len(self.options.include_suites) or len(self.options.exclude_suites):
+            tests = []
+            for tst in self.tests:
+                if self.test_suitable(tst):
+                    tests.append(tst)
+        else:
+            tests = self.tests
 
         if self.options.args:
             tests = [t for t in tests if t.name in self.options.args]
@@ -384,8 +428,15 @@ def get_wrapper(self):
         assert(isinstance(wrap, list))
         return wrap
 
-    def get_suites(self, tests):
-        return set([test.suite[0] for test in tests])
+    def get_pretty_suite(self, test, tests):
+        if len(self.suites) > 1:
+            rv = TestHarness.split_suite_string(test.suite[0])[0]
+            s = "+".join(TestHarness.split_suite_string(s)[1] for s in test.suite)
+            if len(s):
+                rv += ":"
+            return rv + s + " / " + test.name
+        else:
+            return test.name
 
     def run_tests(self, tests):
         try:
@@ -399,13 +450,7 @@ def run_tests(self, tests):
 
             for i in range(self.options.repeat):
                 for i, test in enumerate(tests):
-                    if test.suite[0] == '':
-                        visible_name = test.name
-                    else:
-                        if self.options.suite is not None:
-                            visible_name = self.options.suite + ' / ' + test.name
-                        else:
-                            visible_name = test.suite[0] + ' / ' + test.name
+                    visible_name = self.get_pretty_suite(test, tests)
 
                     if self.options.gdb:
                         test.timeout = None
@@ -461,13 +506,8 @@ def run_special(self):
 
 def list_tests(th):
     tests = th.get_tests()
-    print_suites = True if len(th.get_suites(tests)) != 1 else False
-    for i in tests:
-        if print_suites:
-            print("%s / %s" % (i.suite[0], i.name))
-        else:
-            print("%s" % i.name)
-
+    for t in tests:
+        print(th.get_pretty_suite(t, tests))
 
 def merge_suite_options(options):
     buildfile = os.path.join(options.wd, 'meson-private/build.dat')
diff --git a/run_unittests.py b/run_unittests.py
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -418,6 +418,61 @@ def test_testsetups(self):
         self.assertTrue('TEST_ENV is set' in vg_log)
         self.assertTrue('Memcheck' in vg_log)
 
+    def assertFailedTestCount(self, failure_count, command):
+        try:
+            self._run(command)
+            self.assertEqual(0, failure_count, 'Expected %d tests to fail.' % failure_count)
+        except subprocess.CalledProcessError as e:
+            self.assertEqual(e.returncode, failure_count)
+
+    def test_suite_selection(self):
+        testdir = os.path.join(self.unit_test_dir, '4 suite selection')
+        self.init(testdir)
+        self.build()
+
+        self.assertFailedTestCount(3, self.mtest_command)
+
+        self.assertFailedTestCount(0, self.mtest_command + ['--suite', ':success'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--suite', ':fail'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', ':success'])
+        self.assertFailedTestCount(0, self.mtest_command + ['--no-suite', ':fail'])
+
+        self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'mainprj'])
+        self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjsucc'])
+        self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjfail'])
+        self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjmix'])
+        self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'mainprj'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjsucc'])
+        self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'subprjfail'])
+        self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'subprjmix'])
+
+        self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'mainprj:fail'])
+        self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'mainprj:success'])
+        self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'mainprj:fail'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'mainprj:success'])
+
+        self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjfail:fail'])
+        self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjfail:success'])
+        self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'subprjfail:fail'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjfail:success'])
+
+        self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjsucc:fail'])
+        self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjsucc:success'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjsucc:fail'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjsucc:success'])
+
+        self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjmix:fail'])
+        self.assertFailedTestCount(0, self.mtest_command + ['--suite', 'subprjmix:success'])
+        self.assertFailedTestCount(2, self.mtest_command + ['--no-suite', 'subprjmix:fail'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--no-suite', 'subprjmix:success'])
+
+        self.assertFailedTestCount(2, self.mtest_command + ['--suite', 'subprjfail', '--suite', 'subprjmix:fail'])
+        self.assertFailedTestCount(3, self.mtest_command + ['--suite', 'subprjfail', '--suite', 'subprjmix', '--suite', 'mainprj'])
+        self.assertFailedTestCount(2, self.mtest_command + ['--suite', 'subprjfail', '--suite', 'subprjmix', '--suite', 'mainprj', '--no-suite', 'subprjmix:fail'])
+        self.assertFailedTestCount(1, self.mtest_command + ['--suite', 'subprjfail', '--suite', 'subprjmix', '--suite', 'mainprj', '--no-suite', 'subprjmix:fail', 'mainprj-failing_test'])
+
+        self.assertFailedTestCount(1, self.mtest_command + ['--no-suite', 'subprjfail:fail', '--no-suite', 'subprjmix:fail'])
+
     def _test_stds_impl(self, testdir, compiler, p):
         lang_std = p + '_std'
         # Check that all the listed -std=xxx options for this compiler work


Test patch:
diff --git a/test cases/unit/4 suite selection/failing_test.c b/test cases/unit/4 suite selection/failing_test.c
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/failing_test.c	
@@ -0,0 +1 @@
+int main() { return -1 ; }
diff --git a/test cases/unit/4 suite selection/meson.build b/test cases/unit/4 suite selection/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/meson.build	
@@ -0,0 +1,13 @@
+project('mainprj', 'c')
+
+subproject('subprjfail')
+subproject('subprjsucc')
+subproject('subprjmix')
+
+test('mainprj-failing_test',
+  executable('failing_test', 'failing_test.c'),
+  suite : 'fail')
+
+test('mainprj-successful_test',
+  executable('successful_test', 'successful_test.c'),
+  suite : 'success')
diff --git a/test cases/unit/4 suite selection/subprojects/subprjfail/failing_test.c b/test cases/unit/4 suite selection/subprojects/subprjfail/failing_test.c
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/subprojects/subprjfail/failing_test.c	
@@ -0,0 +1 @@
+int main() { return -1 ; }
diff --git a/test cases/unit/4 suite selection/subprojects/subprjfail/meson.build b/test cases/unit/4 suite selection/subprojects/subprjfail/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/subprojects/subprjfail/meson.build	
@@ -0,0 +1,5 @@
+project('subprjfail', 'c')
+
+test('subprjfail-failing_test',
+  executable('failing_test', 'failing_test.c'),
+  suite : 'fail')
diff --git a/test cases/unit/4 suite selection/subprojects/subprjmix/failing_test.c b/test cases/unit/4 suite selection/subprojects/subprjmix/failing_test.c
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/subprojects/subprjmix/failing_test.c	
@@ -0,0 +1 @@
+int main() { return -1 ; }
diff --git a/test cases/unit/4 suite selection/subprojects/subprjmix/meson.build b/test cases/unit/4 suite selection/subprojects/subprjmix/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/subprojects/subprjmix/meson.build	
@@ -0,0 +1,9 @@
+project('subprjmix', 'c')
+
+test('subprjmix-failing_test',
+  executable('failing_test', 'failing_test.c'),
+  suite : 'fail')
+
+test('subprjmix-successful_test',
+  executable('successful_test', 'successful_test.c'),
+  suite : 'success')
diff --git a/test cases/unit/4 suite selection/subprojects/subprjmix/successful_test.c b/test cases/unit/4 suite selection/subprojects/subprjmix/successful_test.c
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/subprojects/subprjmix/successful_test.c	
@@ -0,0 +1 @@
+int main() { return 0 ; }
diff --git a/test cases/unit/4 suite selection/subprojects/subprjsucc/meson.build b/test cases/unit/4 suite selection/subprojects/subprjsucc/meson.build
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/subprojects/subprjsucc/meson.build	
@@ -0,0 +1,5 @@
+project('subprjsucc', 'c')
+
+test('subprjsucc-successful_test',
+  executable('successful_test', 'successful_test.c'),
+  suite : 'success')
diff --git a/test cases/unit/4 suite selection/subprojects/subprjsucc/successful_test.c b/test cases/unit/4 suite selection/subprojects/subprjsucc/successful_test.c
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/subprojects/subprjsucc/successful_test.c	
@@ -0,0 +1 @@
+int main() { return 0 ; }
diff --git a/test cases/unit/4 suite selection/successful_test.c b/test cases/unit/4 suite selection/successful_test.c
new file mode 100644
--- /dev/null
+++ b/test cases/unit/4 suite selection/successful_test.c	
@@ -0,0 +1 @@
+int main() { return 0 ; }