Repository: randombit/botan
Pull request #3215 (2023-01-24 08:27:52)

Chore: Improve Test Registration
This introduces an internal `Test_Registry` class to centralize test case registration and filtering.

Mostly, this allows to filter tests by category like: `./botan-test tls` would run all tests related to TLS. The category annotation in `BOTAN_REGISTER_TEST(category, test_name, ...)` did exist but was unused so far. To do that in a user-friendly way, category names and test names need to be disjoint.

Test filters can be mixed (i.e. some categories and some specific test names). Skipping specific tests within a category can be done as before using `--skip-tests`. There's no such thing as `--skip-category`, yet. Note that filtering is implemented fairly naively and inefficiently (with a bunch of lists), to keep it simple. But probably `O(n^2)` is fine for mangling a list of tests. 🤡 

Currently available test categories for reference:

```
Available test categories
----------------
asn1 block codec compat compression dilithium ec_h2c ffi filters
hash kdf keywrap kyber mac math misc modes otp pake pbkdf pkcs11
pubkey rng roughtime stream tls tls_extensions utils x509 zfec
```

Technically, "kyber" and "dilithium" should move into "pubkey", I guess. Or maybe "PQC"?

While changing a few test names, I started seeing race conditions in multi-threaded execution: Turns out that there were centralized lists of test names to identify "smoke tests" and "serialized tests" hidden in the test runner.
To avoid this tight coupling between test names and runner behavior, tests are now annotated accordingly during registration using `BOTAN_REGISTER_SERIALIZED_TEST` and `BOTAN_REGISTER_SMOKE_TEST`.

The above-described changes are split into individual commits for easier review. If needed, we could also split it into two or three PRs.


Patch:
diff --git a/src/lib/utils/stl_util.h b/src/lib/utils/stl_util.h
index 9d89c2988..4d146be10 100644
--- a/src/lib/utils/stl_util.h
+++ b/src/lib/utils/stl_util.h
@@ -44,6 +44,20 @@ std::set<K> map_keys_as_set(const std::map<K, V>& kv)
    return s;
    }
 
+/**
+* Return the keys of a multimap as a std::set
+*/
+template<typename K, typename V>
+std::set<K> map_keys_as_set(const std::multimap<K, V>& kv)
+   {
+   std::set<K> s;
+   for(auto&& i : kv)
+      {
+      s.insert(i.first);
+      }
+   return s;
+   }
+
 /*
 * Searching through a std::map
 * @param mapping the map to search


Test patch:
diff --git a/src/tests/main.cpp b/src/tests/main.cpp
index c940b7d65..e0daccd04 100644
--- a/src/tests/main.cpp
+++ b/src/tests/main.cpp
@@ -16,6 +16,28 @@
 
 namespace {
 
+void print_item_list(std::ostringstream& err, std::set<std::string> list)
+   {
+   size_t line_len = 0;
+
+   for(auto const& item : list)
+         {
+         err << item << " ";
+         line_len += item.size() + 1;
+
+         if(line_len > 64)
+            {
+            err << "\n";
+            line_len = 0;
+            }
+         }
+
+      if(line_len > 0)
+         {
+         err << "\n";
+         }
+   }
+
 std::string help_text(const std::string& spec)
    {
    std::ostringstream err;
@@ -24,24 +46,13 @@ std::string help_text(const std::string& spec)
        << "Available test suites\n"
        << "----------------\n";
 
-   size_t line_len = 0;
-
-   for(auto const& test : Botan_Tests::Test::registered_tests())
-      {
-      err << test << " ";
-      line_len += test.size() + 1;
+   print_item_list(err, Botan_Tests::Test::registered_tests());
 
-      if(line_len > 64)
-         {
-         err << "\n";
-         line_len = 0;
-         }
-      }
+   err << '\n'
+       << "Available test categories\n"
+       << "----------------\n";
 
-   if(line_len > 0)
-      {
-      err << "\n";
-      }
+   print_item_list(err, Botan_Tests::Test::registered_test_categories());
 
    return err.str();
    }
diff --git a/src/tests/runner/test_runner.cpp b/src/tests/runner/test_runner.cpp
index c8487c2db..50129dd6c 100644
--- a/src/tests/runner/test_runner.cpp
+++ b/src/tests/runner/test_runner.cpp
@@ -87,9 +87,6 @@ class Testsuite_RNG final : public Botan::RandomNumberGenerator
 
 bool Test_Runner::run(const Test_Options& opts)
    {
-   std::vector<std::string> req = opts.requested_tests();
-   const std::set<std::string>& to_skip = opts.skip_tests();
-
    m_reporters.emplace_back(std::make_unique<StdoutReporter>(opts, output()));
    if(!opts.xml_results_dir().empty())
       {
@@ -100,73 +97,28 @@ bool Test_Runner::run(const Test_Options& opts)
 #endif
       }
 
-   if(req.empty())
-      {
-      /*
-      If nothing was requested on the command line, run everything. First
-      run the "essentials" to smoke test, then everything else in
-      alphabetical order.
-      */
-
-      std::vector<std::string> default_first = {
-         "block", "stream", "hash", "mac", "aead",
-         "modes", "kdf", "pbkdf", "hmac_drbg", "util"
-      };
-
-      for(const auto& s : default_first)
-         {
-         if(to_skip.count(s) == 0)
-            req.push_back(s);
-         }
+   auto req = Botan_Tests::Test::filter_registered_tests(opts.requested_tests(), opts.skip_tests());
 
-      std::set<std::string> all_others = Botan_Tests::Test::registered_tests();
-
-      if(opts.pkcs11_lib().empty())
+   // TODO: Test runner should not be aware of certain test's environmental requirements.
+   if(opts.pkcs11_lib().empty())
+      {
+      // do not run pkcs11 tests by default unless pkcs11-lib set
+      for(auto iter = req.begin(); iter != req.end();)
          {
-         // do not run pkcs11 tests by default unless pkcs11-lib set
-         for(std::set<std::string>::iterator iter = all_others.begin(); iter != all_others.end();)
+         if((*iter).find("pkcs11") != std::string::npos)
             {
-            if((*iter).find("pkcs11") != std::string::npos)
-               {
-               iter = all_others.erase(iter);
-               }
-            else
-               {
-               ++iter;
-               }
+            iter = req.erase(iter);
             }
-         }
-
-      for(const auto& f : req)
-         {
-         all_others.erase(f);
-         }
-
-      for(const std::string& f : to_skip)
-         {
-         all_others.erase(f);
-         }
-
-      req.insert(req.end(), all_others.begin(), all_others.end());
-      }
-   else if(req.size() == 1 && req.at(0) == "pkcs11")
-      {
-      req = {"pkcs11-manage", "pkcs11-module", "pkcs11-slot", "pkcs11-session", "pkcs11-object", "pkcs11-rsa",
-             "pkcs11-ecdsa", "pkcs11-ecdh", "pkcs11-rng", "pkcs11-x509"
-      };
-      }
-   else
-      {
-      std::set<std::string> all = Botan_Tests::Test::registered_tests();
-      for(auto const& r : req)
-         {
-         if(all.find(r) == all.end())
+         else
             {
-            throw Botan_Tests::Test_Error("Unknown test suite: " + r);
+            ++iter;
             }
          }
       }
 
+   if(req.empty())
+      { throw Test_Error("No tests to run"); }
+
    std::vector<uint8_t> seed = Botan::hex_decode(opts.drbg_seed());
    if(seed.empty())
       {
@@ -263,23 +215,6 @@ std::vector<Test::Result> run_a_test(const std::string& test_name)
    return results;
    }
 
-#if defined(BOTAN_HAS_THREAD_UTILS)
-
-bool needs_serialization(const std::string& test_name)
-   {
-   if(test_name.substr(0, 6) == "pkcs11")
-      return true;
-   if(test_name == "block" || test_name == "hash" || test_name == "mac" || test_name == "stream" || test_name == "aead")
-      return true;
-   if(test_name == "argon2")
-      return true;
-   if(test_name == "ecc_unit")
-      return false;
-   return false;
-   }
-
-#endif
-
 bool all_passed(const std::vector<Test::Result>& results)
    {
    return std::all_of(results.begin(), results.end(),
@@ -316,7 +251,7 @@ bool Test_Runner::run_tests_multithreaded(const std::vector<std::string>& tests_
 
    for(auto const& test_name : tests_to_run)
       {
-      if(needs_serialization(test_name))
+      if(Test::test_needs_serialization(test_name))
          {
          m_fut_results.push_back(pool.run(run_test_exclusive, test_name));
          }
diff --git a/src/tests/test_aead.cpp b/src/tests/test_aead.cpp
index 271b84521..48fa8d660 100644
--- a/src/tests/test_aead.cpp
+++ b/src/tests/test_aead.cpp
@@ -485,7 +485,7 @@ class AEAD_Tests final : public Text_Based_Test
          }
    };
 
-BOTAN_REGISTER_TEST("modes", "aead", AEAD_Tests);
+BOTAN_REGISTER_SERIALIZED_SMOKE_TEST("modes", "aead", AEAD_Tests);
 
 #endif
 
diff --git a/src/tests/test_asn1.cpp b/src/tests/test_asn1.cpp
index bbd0cf009..9414975a9 100644
--- a/src/tests/test_asn1.cpp
+++ b/src/tests/test_asn1.cpp
@@ -313,7 +313,7 @@ class ASN1_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("asn1", "asn1", ASN1_Tests);
+BOTAN_REGISTER_TEST("asn1", "asn1_encoding", ASN1_Tests);
 
 class ASN1_Time_Parsing_Tests final : public Text_Based_Test
    {
diff --git a/src/tests/test_block.cpp b/src/tests/test_block.cpp
index d48f0554a..8d0254d15 100644
--- a/src/tests/test_block.cpp
+++ b/src/tests/test_block.cpp
@@ -196,7 +196,7 @@ class Block_Cipher_Tests final : public Text_Based_Test
 
    };
 
-BOTAN_REGISTER_TEST("block", "block", Block_Cipher_Tests);
+BOTAN_REGISTER_SERIALIZED_SMOKE_TEST("block", "block_ciphers", Block_Cipher_Tests);
 
 }
 
diff --git a/src/tests/test_compression.cpp b/src/tests/test_compression.cpp
index bc21fbeb6..b60b02118 100644
--- a/src/tests/test_compression.cpp
+++ b/src/tests/test_compression.cpp
@@ -175,7 +175,7 @@ class Compression_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("compression", "compression", Compression_Tests);
+BOTAN_REGISTER_TEST("compression", "compression_tests", Compression_Tests);
 
 class CompressionCreate_Tests final : public Test
    {
diff --git a/src/tests/test_hash.cpp b/src/tests/test_hash.cpp
index 24ae276a1..427c235e0 100644
--- a/src/tests/test_hash.cpp
+++ b/src/tests/test_hash.cpp
@@ -35,7 +35,7 @@ class Invalid_Hash_Name_Tests final : public Test
    private:
       static void test_invalid_name(Result& result,
                              const std::string& name,
-                             const std::string& expected_msg = "") 
+                             const std::string& expected_msg = "")
          {
          try
             {
@@ -179,7 +179,7 @@ class Hash_Function_Tests final : public Text_Based_Test
 
    };
 
-BOTAN_REGISTER_TEST("hash", "hash", Hash_Function_Tests);
+BOTAN_REGISTER_SERIALIZED_SMOKE_TEST("hash", "hash_algos", Hash_Function_Tests);
 
 class Hash_NIST_MonteCarlo_Tests final : public Text_Based_Test
    {
diff --git a/src/tests/test_kdf.cpp b/src/tests/test_kdf.cpp
index 3104a66f6..a46bdf422 100644
--- a/src/tests/test_kdf.cpp
+++ b/src/tests/test_kdf.cpp
@@ -55,7 +55,7 @@ class KDF_KAT_Tests final : public Text_Based_Test
 
    };
 
-BOTAN_REGISTER_TEST("kdf", "kdf", KDF_KAT_Tests);
+BOTAN_REGISTER_SMOKE_TEST("kdf", "kdf_kat", KDF_KAT_Tests);
 
 #endif
 
diff --git a/src/tests/test_mac.cpp b/src/tests/test_mac.cpp
index 3955ea7b5..6b2698cef 100644
--- a/src/tests/test_mac.cpp
+++ b/src/tests/test_mac.cpp
@@ -162,7 +162,7 @@ class Message_Auth_Tests final : public Text_Based_Test
          }
    };
 
-BOTAN_REGISTER_TEST("mac", "mac", Message_Auth_Tests);
+BOTAN_REGISTER_SERIALIZED_SMOKE_TEST("mac", "mac_algos", Message_Auth_Tests);
 
 #endif
 
diff --git a/src/tests/test_modes.cpp b/src/tests/test_modes.cpp
index 0ec4f88e7..fae96d251 100644
--- a/src/tests/test_modes.cpp
+++ b/src/tests/test_modes.cpp
@@ -264,7 +264,7 @@ class Cipher_Mode_Tests final : public Text_Based_Test
          }
    };
 
-BOTAN_REGISTER_TEST("modes", "modes", Cipher_Mode_Tests);
+BOTAN_REGISTER_SMOKE_TEST("modes", "cipher_modes", Cipher_Mode_Tests);
 
 class Cipher_Mode_IV_Carry_Tests final : public Test
    {
diff --git a/src/tests/test_pbkdf.cpp b/src/tests/test_pbkdf.cpp
index d6b242fe3..3b417b50e 100644
--- a/src/tests/test_pbkdf.cpp
+++ b/src/tests/test_pbkdf.cpp
@@ -71,7 +71,7 @@ class PBKDF_KAT_Tests final : public Text_Based_Test
 
    };
 
-BOTAN_REGISTER_TEST("pbkdf", "pbkdf", PBKDF_KAT_Tests);
+BOTAN_REGISTER_SMOKE_TEST("pbkdf", "pbkdf_kat", PBKDF_KAT_Tests);
 
 class Pwdhash_Tests : public Test
    {
@@ -286,7 +286,7 @@ class Argon2_KAT_Tests final : public Text_Based_Test
 
    };
 
-BOTAN_REGISTER_TEST("pbkdf", "argon2", Argon2_KAT_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pbkdf", "argon2", Argon2_KAT_Tests);
 
 #endif
 
diff --git a/src/tests/test_pkcs11_high_level.cpp b/src/tests/test_pkcs11_high_level.cpp
index b82f3d3a1..0ccf2dc73 100644
--- a/src/tests/test_pkcs11_high_level.cpp
+++ b/src/tests/test_pkcs11_high_level.cpp
@@ -200,7 +200,7 @@ class Module_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-module", Module_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-module", Module_Tests);
 
 /***************************** Slot *****************************/
 
@@ -339,7 +339,7 @@ class Slot_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-slot", Slot_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-slot", Slot_Tests);
 
 /***************************** Session *****************************/
 
@@ -471,7 +471,7 @@ class Session_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-session", Session_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-session", Session_Tests);
 
 /***************************** Object *****************************/
 
@@ -660,7 +660,7 @@ class Object_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-object", Object_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-object", Object_Tests);
 
 /***************************** PKCS11 RSA *****************************/
 
@@ -941,7 +941,7 @@ class PKCS11_RSA_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-rsa", PKCS11_RSA_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-rsa", PKCS11_RSA_Tests);
 #endif
 
 /***************************** PKCS11 ECDSA *****************************/
@@ -1233,7 +1233,7 @@ class PKCS11_ECDSA_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-ecdsa", PKCS11_ECDSA_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-ecdsa", PKCS11_ECDSA_Tests);
 
 #endif
 
@@ -1455,7 +1455,7 @@ class PKCS11_ECDH_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-ecdh", PKCS11_ECDH_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-ecdh", PKCS11_ECDH_Tests);
 
 #endif
 
@@ -1541,7 +1541,7 @@ class PKCS11_RNG_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-rng", PKCS11_RNG_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-rng", PKCS11_RNG_Tests);
 
 /***************************** PKCS11 token management *****************************/
 
@@ -1626,7 +1626,7 @@ class PKCS11_Token_Management_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-manage", PKCS11_Token_Management_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-manage", PKCS11_Token_Management_Tests);
 
 /***************************** PKCS11 token management *****************************/
 
@@ -1670,7 +1670,7 @@ class PKCS11_X509_Tests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-x509", PKCS11_X509_Tests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-x509", PKCS11_X509_Tests);
 
 #endif
 
diff --git a/src/tests/test_pkcs11_low_level.cpp b/src/tests/test_pkcs11_low_level.cpp
index 2e147de5b..eacee276a 100644
--- a/src/tests/test_pkcs11_low_level.cpp
+++ b/src/tests/test_pkcs11_low_level.cpp
@@ -834,7 +834,7 @@ class LowLevelTests final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("pkcs11", "pkcs11-lowlevel", LowLevelTests);
+BOTAN_REGISTER_SERIALIZED_TEST("pkcs11", "pkcs11-lowlevel", LowLevelTests);
 
 #endif
 #endif
diff --git a/src/tests/test_rng_kat.cpp b/src/tests/test_rng_kat.cpp
index e38c9a3cb..5020f6799 100644
--- a/src/tests/test_rng_kat.cpp
+++ b/src/tests/test_rng_kat.cpp
@@ -65,7 +65,7 @@ class HMAC_DRBG_Tests final : public Text_Based_Test
 
    };
 
-BOTAN_REGISTER_TEST("rng", "hmac_drbg", HMAC_DRBG_Tests);
+BOTAN_REGISTER_SMOKE_TEST("rng", "hmac_drbg", HMAC_DRBG_Tests);
 
 #endif
 
diff --git a/src/tests/test_roughtime.cpp b/src/tests/test_roughtime.cpp
index 063b59529..9d15b9c55 100644
--- a/src/tests/test_roughtime.cpp
+++ b/src/tests/test_roughtime.cpp
@@ -256,7 +256,7 @@ class Roughtime final : public Test
          }
    };
 
-BOTAN_REGISTER_TEST("roughtime", "roughtime", Roughtime);
+BOTAN_REGISTER_TEST("roughtime", "roughtime_tests", Roughtime);
 
 #endif
 
diff --git a/src/tests/test_stream.cpp b/src/tests/test_stream.cpp
index f5a819b64..175c4e1b0 100644
--- a/src/tests/test_stream.cpp
+++ b/src/tests/test_stream.cpp
@@ -224,7 +224,7 @@ class Stream_Cipher_Tests final : public Text_Based_Test
          }
    };
 
-BOTAN_REGISTER_TEST("stream", "stream", Stream_Cipher_Tests);
+BOTAN_REGISTER_SERIALIZED_SMOKE_TEST("stream", "stream_ciphers", Stream_Cipher_Tests);
 
 #endif
 
diff --git a/src/tests/test_tpm.cpp b/src/tests/test_tpm.cpp
index 2d8fd0684..c5ebbc3fe 100644
--- a/src/tests/test_tpm.cpp
+++ b/src/tests/test_tpm.cpp
@@ -79,7 +79,7 @@ class TPM_Tests final : public Test
 
    };
 
-BOTAN_REGISTER_TEST("tpm", "tpm", TPM_Tests);
+BOTAN_REGISTER_TEST("tpm", "tpm_tests", TPM_Tests);
 
 class UUID_Tests final : public Test
    {
diff --git a/src/tests/test_utils.cpp b/src/tests/test_utils.cpp
index 7886febd4..3efc76beb 100644
--- a/src/tests/test_utils.cpp
+++ b/src/tests/test_utils.cpp
@@ -183,7 +183,7 @@ class Utility_Function_Tests final : public Text_Based_Test
          }
    };
 
-BOTAN_REGISTER_TEST("utils", "util", Utility_Function_Tests);
+BOTAN_REGISTER_SMOKE_TEST("utils", "util", Utility_Function_Tests);
 
 class CT_Mask_Tests final : public Test
    {
diff --git a/src/tests/tests.cpp b/src/tests/tests.cpp
index 342c9b8a6..5a158b4ba 100644
--- a/src/tests/tests.cpp
+++ b/src/tests/tests.cpp
@@ -500,24 +500,134 @@ std::string Test::Result::result_string() const
    return report.str();
    }
 
-// static Test:: functions
-//static
-std::map<std::string, std::function<std::unique_ptr<Test> ()>>& Test::global_registry()
+namespace {
+
+class Test_Registry
    {
-   static std::map<std::string, std::function<std::unique_ptr<Test> ()>> g_test_registry;
-   return g_test_registry;
-   }
+   public:
+      static Test_Registry& instance()
+         {
+         static Test_Registry registry;
+         return registry;
+         }
+
+      void register_test(std::string category,
+                         std::string name,
+                         bool smoke_test,
+                         bool needs_serialization,
+                         std::function<std::unique_ptr<Test>()> maker_fn)
+         {
+         if(m_tests.count(name) != 0)
+            throw Test_Error("Duplicate registration of test '" + name + "'");
+
+         if(m_tests.count(category))
+            throw Test_Error("'" + category + "' cannot be used as category, test exists");
+
+         if(m_categories.count(name))
+            throw Test_Error("'" + name + "' cannot be used as test name, category exists");
+
+         if(smoke_test)
+            m_smoke_tests.push_back(name);
+
+         if(needs_serialization)
+            m_mutexed_tests.push_back(name);
+
+         m_tests.emplace(name, std::move(maker_fn));
+         m_categories.emplace(std::move(category), std::move(name));
+         }
+
+      std::unique_ptr<Test> get_test(const std::string& test_name) const
+         {
+         auto i = m_tests.find(test_name);
+         if(i != m_tests.end())
+            {
+            return i->second();
+            }
+         return nullptr;
+         }
+
+      std::set<std::string> registered_tests() const
+         {
+         return Botan::map_keys_as_set(m_tests);
+         }
+
+      std::set<std::string> registered_test_categories() const
+         {
+         return Botan::map_keys_as_set(m_categories);
+         }
+
+      std::vector<std::string> filter_registered_tests(const std::vector<std::string>& requested, const std::set<std::string>& to_be_skipped)
+         {
+         std::vector<std::string> result;
+
+         // TODO: this is O(n^2), but we have a relatively small number of tests.
+         auto insert_if_not_exists_and_not_skipped = [&](const std::string test_name)
+            {
+            if(!Botan::value_exists(result, test_name) && to_be_skipped.find(test_name) == to_be_skipped.end())
+               result.push_back(test_name);
+            };
+
+         if(requested.empty())
+            {
+            /*
+            If nothing was requested on the command line, run everything. First
+            run the "essentials" to smoke test, then everything else in
+            alphabetical order.
+            */
+            result = m_smoke_tests;
+            for(const auto& [test_name, _] : m_tests)
+               { insert_if_not_exists_and_not_skipped(test_name); }
+            }
+         else
+            {
+            for(const auto& r : requested)
+               {
+               if(m_tests.find(r) != m_tests.end())
+                  { insert_if_not_exists_and_not_skipped(r); }
+               else if(auto elems = m_categories.equal_range(r); elems.first != m_categories.end())
+                  {
+                  for(;elems.first != elems.second; ++elems.first)
+                     {
+                     insert_if_not_exists_and_not_skipped(elems.first->second);
+                     }
+                  }
+               else
+                  {
+                  throw Botan_Tests::Test_Error("Unknown test suite or category: " + r);
+                  }
+               }
+            }
+
+         return result;
+         }
+
+      bool needs_serialization(const std::string& test_name) const
+         {
+         return Botan::value_exists(m_mutexed_tests, test_name);
+         }
+
+   private:
+      Test_Registry() = default;
+
+   private:
+      std::map<std::string, std::function<std::unique_ptr<Test> ()>> m_tests;
+      std::multimap<std::string, std::string> m_categories;
+      std::vector<std::string> m_smoke_tests;
+      std::vector<std::string> m_mutexed_tests;
+   };
+
+}
+
+// static Test:: functions
 
 //static
-void Test::register_test(const std::string& category,
-                         const std::string& name,
-                         const std::function<std::unique_ptr<Test> ()>& maker_fn)
+void Test::register_test(std::string category,
+                         std::string name,
+                         bool smoke_test,
+                         bool needs_serialization,
+                         std::function<std::unique_ptr<Test> ()> maker_fn)
    {
-   BOTAN_UNUSED(category);
-   if(Test::global_registry().count(name) != 0)
-      throw Test_Error("Duplicate registration of test '" + name + "'");
-
-   Test::global_registry().insert(std::make_pair(name, maker_fn));
+   Test_Registry::instance().register_test(std::move(category), std::move(name), smoke_test, needs_serialization, std::move(maker_fn));
    }
 
 //static
@@ -530,18 +640,33 @@ uint64_t Test::timestamp()
 //static
 std::set<std::string> Test::registered_tests()
    {
-   return Botan::map_keys_as_set(Test::global_registry());
+   return Test_Registry::instance().registered_tests();
+   }
+
+
+//static
+std::set<std::string> Test::registered_test_categories()
+   {
+   return Test_Registry::instance().registered_test_categories();
    }
 
 //static
 std::unique_ptr<Test> Test::get_test(const std::string& test_name)
    {
-   auto i = Test::global_registry().find(test_name);
-   if(i != Test::global_registry().end())
-      {
-      return std::unique_ptr<Test>(i->second());
-      }
-   return nullptr;
+   return Test_Registry::instance().get_test(test_name);
+   }
+
+//static
+bool Test::test_needs_serialization(const std::string& test_name)
+   {
+   return Test_Registry::instance().needs_serialization(test_name);
+   }
+
+//static
+std::vector<std::string> Test::filter_registered_tests(const std::vector<std::string>& requested,
+                                                       const std::set<std::string>& to_be_skipped)
+   {
+   return Test_Registry::instance().filter_registered_tests(requested, to_be_skipped);
    }
 
 //static
diff --git a/src/tests/tests.h b/src/tests/tests.h
index 4c8c8e61a..4ed5967f3 100644
--- a/src/tests/tests.h
+++ b/src/tests/tests.h
@@ -641,15 +641,20 @@ class Test
       void set_registration_location(CodeLocation location) { m_registration_location = std::move(location); }
       const std::optional<CodeLocation>& registration_location() const { return m_registration_location; }
 
-      static void register_test(const std::string& category,
-                                const std::string& name,
-                                const std::function<std::unique_ptr<Test> ()>& maker_fn);
-
-      static std::map<std::string, std::function<std::unique_ptr<Test> ()>>& global_registry();
+      /// @p smoke_test are run first in an unfiltered test run
+      static void register_test(std::string category,
+                                std::string name,
+                                bool smoke_test,
+                                bool needs_serialization,
+                                std::function<std::unique_ptr<Test> ()> maker_fn);
 
       static std::set<std::string> registered_tests();
+      static std::set<std::string> registered_test_categories();
+      static std::vector<std::string> filter_registered_tests(const std::vector<std::string>& requested,
+                                                              const std::set<std::string>& to_be_skipped);
 
       static std::unique_ptr<Test> get_test(const std::string& test_name);
+      static bool test_needs_serialization(const std::string& test_name);
 
       static std::string data_file(const std::string& what);
 
@@ -723,20 +728,25 @@ template<typename Test_Class>
 class TestClassRegistration
    {
    public:
-      TestClassRegistration(std::string category, std::string name, CodeLocation registration_location)
+      TestClassRegistration(std::string category, std::string name, bool smoke_test, bool needs_serialization, CodeLocation registration_location)
          {
-         auto test_maker = [=]() -> std::unique_ptr<Test>
+         Test::register_test(std::move(category), std::move(name), smoke_test, needs_serialization, [=]
             {
             auto test = std::make_unique<Test_Class>();
-            test->set_registration_location(std::move(registration_location));
+            test->set_registration_location(registration_location);
             return test;
-            };
-         Test::register_test(category, name, test_maker);
+            });
          }
    };
 
 #define BOTAN_REGISTER_TEST(category, name, Test_Class)                 \
-   const TestClassRegistration<Test_Class> reg_ ## Test_Class ## _tests(category, name, {__FILE__, __LINE__})
+   const TestClassRegistration<Test_Class> reg_ ## Test_Class ## _tests(category, name, false, false, {__FILE__, __LINE__})
+#define BOTAN_REGISTER_SERIALIZED_TEST(category, name, Test_Class)                 \
+   const TestClassRegistration<Test_Class> reg_ ## Test_Class ## _tests(category, name, false, true, {__FILE__, __LINE__})
+#define BOTAN_REGISTER_SMOKE_TEST(category, name, Test_Class)           \
+   const TestClassRegistration<Test_Class> reg_ ## Test_Class ## _tests(category, name, true, false, {__FILE__, __LINE__})
+#define BOTAN_REGISTER_SERIALIZED_SMOKE_TEST(category, name, Test_Class)           \
+   const TestClassRegistration<Test_Class> reg_ ## Test_Class ## _tests(category, name, true, true, {__FILE__, __LINE__})
 
 typedef Test::Result(*test_fn)();
 typedef std::vector<Test::Result> (*test_fn_vec)();
@@ -801,20 +811,25 @@ class TestFnRegistration
    {
    public:
       template <typename... TestFns>
-      TestFnRegistration(std::string category, std::string name, CodeLocation registration_location, TestFns... fn)
+      TestFnRegistration(std::string category, std::string name, bool smoke_test, bool needs_serialization, CodeLocation registration_location, TestFns... fn)
          {
-         auto test_maker = [=]() -> std::unique_ptr<Test>
+         Test::register_test(std::move(category), std::move(name), smoke_test, needs_serialization, [=]
             {
             auto test = std::make_unique<FnTest>(fn...);
             test->set_registration_location(std::move(registration_location));
             return test;
-            };
-         Test::register_test(category, name, test_maker);
+            });
          }
    };
 
 #define BOTAN_REGISTER_TEST_FN(category, name, ...) \
-   static const TestFnRegistration reg_ ## fn_name(category, name, {__FILE__, __LINE__}, __VA_ARGS__)
+   static const TestFnRegistration reg_ ## fn_name(category, name, false, false, {__FILE__, __LINE__}, __VA_ARGS__)
+#define BOTAN_REGISTER_SMOKE_TEST_FN(category, name, ...) \
+   static const TestFnRegistration reg_ ## fn_name(category, name, true, false, {__FILE__, __LINE__}, __VA_ARGS__)
+#define BOTAN_REGISTER_SERIALIZED_TEST_FN(category, name, ...) \
+   static const TestFnRegistration reg_ ## fn_name(category, name, false, true {__FILE__, __LINE__}, __VA_ARGS__)
+#define BOTAN_REGISTER_SERIALIZED_SMOKE_TEST_FN(category, name, ...) \
+   static const TestFnRegistration reg_ ## fn_name(category, name, true, true {__FILE__, __LINE__}, __VA_ARGS__)
 
 class VarMap
    {
diff --git a/src/tests/unit_tls.cpp b/src/tests/unit_tls.cpp
index 29d64346b..7c81e84b6 100644
--- a/src/tests/unit_tls.cpp
+++ b/src/tests/unit_tls.cpp
@@ -918,7 +918,7 @@ class TLS_Unit_Tests final : public Test
 
    };
 
-BOTAN_REGISTER_TEST("tls", "tls", TLS_Unit_Tests);
+BOTAN_REGISTER_TEST("tls", "unit_tls", TLS_Unit_Tests);
 
 class DTLS_Reconnection_Test : public Test
    {