Repository: randombit/botan
Pull request #2115 (2019-09-24 13:10:34)

Improve ECC PKCS#11 tests
* try catch in p11 high and low level tests. If a test throws an exception, the test output contains the name of the test case that failed. Before, only the name of the suite was printed.
* add a test case that imports elliptic curve parameters
* add curve brainpool512r1 to PKCS11 test test_ecdsa_sign_verify()
* add curve brainpool512r1 to test_ecdsa_generate_keypair()
* update tested/supported smartcards in manual


Patch:
diff --git a/doc/api_ref/pkcs11.rst b/doc/api_ref/pkcs11.rst
index 781777264..9e0ddfa67 100644
--- a/doc/api_ref/pkcs11.rst
+++ b/doc/api_ref/pkcs11.rst
@@ -1281,7 +1281,9 @@ Test results
 +-------------------------------------+-------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+
 | CardOS 5.3                          | mostly works                              | Windows 10, 64-bit, version 1709                  | API Version 5.4.9.77 (Cryptoki v2.11)             |  2.4.0, Cryptoki v2.40                            | [52]_                                             |
 +-------------------------------------+-------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+
-| Gemalto IDPrime MD 3840             | mostly works                              | Windows 10, 64-bit, version 1709                  | IDGo 800, v1.2.4 (Cryptoki v2.20)                 |  2.4.0, Cryptoki v2.40                            | [53]_                                             |
+| CardOS 5.3                          | mostly works                              | Windows 10, 64-bit, version 1903                  | API Version 5.5.1 (Cryptoki v2.11)                |  2.12.0 unreleased, Cryptoki v2.40                | [53]_                                             |
++-------------------------------------+-------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+
+| Gemalto IDPrime MD 3840             | mostly works                              | Windows 10, 64-bit, version 1709                  | IDGo 800, v1.2.4 (Cryptoki v2.20)                 |  2.4.0, Cryptoki v2.40                            | [54]_                                             |
 +-------------------------------------+-------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+
 | SoftHSM 2.3.0 (OpenSSL 1.0.2g)      | works                                     | Windows 10, 64-bit, version 1709                  | Cryptoki v2.40                                    |  2.4.0, Cryptoki v2.40                            |                                                   |
 +-------------------------------------+-------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+---------------------------------------------------+
@@ -1360,7 +1362,23 @@ Test results
 
  - rng_add_entropy [5]_
 
-.. [53] Failing operations for Gemalto IDPrime MD 3840
+.. [53] Failing operations for CardOS 5.3 (middelware 5.5.1)
+
+ - ecdh_privkey_export [2]_
+ - ecdh_generate_private_key [35]_
+ - ecdsa_privkey_export [2]_
+ - ecdsa_generate_private_key [36]_
+ - c_copy_object [4]_
+
+ - object_copy [4]_
+
+ - rng_add_entropy [5]_
+
+ - rsa_sign_verify [3]_
+ - rsa_privkey_export [2]_
+ - rsa_generate_private_key [9]_
+
+.. [54] Failing operations for Gemalto IDPrime MD 3840
 
  - session_login_logout [2]_
  - session_info [2]_
@@ -1385,6 +1403,7 @@ Error descriptions
 .. [6] CKM_X9_42_DH_KEY_PAIR_GEN | CKR_DEVICE_ERROR (0x30=48)
 .. [7] CKR_TEMPLATE_INCONSISTENT (0xD1=209)
 .. [8] CKR_ENCRYPTED_DATA_INVALID | CKM_SHA256_RSA_PKCS (0x40=64)
+.. [9] CKR_TEMPLATE_INCOMPLETE (0xD0=208)
 
 .. [20] Test fails due to unsupported copy function (CKR_FUNCTION_NOT_SUPPORTED)
 .. [21] Generating private key for extraction with property extractable fails (CKR_ARGUMENTS_BAD)
@@ -1396,3 +1415,5 @@ Error descriptions
 .. [32] Invalid argument OS2ECP: Unknown format type 155
 .. [33] Invalid argument OS2ECP: Unknown format type 92
 .. [34] Invalid argument OS2ECP: Unknown format type 57
+.. [35] Invalid argument OS2ECP: Unknown format type 82
+.. [36] Invalid argument OS2ECP: Unknown format type 102


Test patch:
diff --git a/src/tests/test_pkcs11.h b/src/tests/test_pkcs11.h
index 176466262..d3922f635 100644
--- a/src/tests/test_pkcs11.h
+++ b/src/tests/test_pkcs11.h
@@ -1,5 +1,6 @@
 /*
 * (C) 2016 Daniel Neus
+* (C) 2019 Michael Boric
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
@@ -7,6 +8,8 @@
 #ifndef BOTAN_TESTS_PKCS11_H_
 #define BOTAN_TESTS_PKCS11_H_
 
+#define STRING_AND_FUNCTION(x) #x, x
+
 #include "tests.h"
 
 #if defined(BOTAN_HAS_PKCS11)
diff --git a/src/tests/test_pkcs11_high_level.cpp b/src/tests/test_pkcs11_high_level.cpp
index 077e0240f..2d6fd947f 100644
--- a/src/tests/test_pkcs11_high_level.cpp
+++ b/src/tests/test_pkcs11_high_level.cpp
@@ -1,6 +1,7 @@
 /*
 * (C) 2016 Daniel Neus
 * (C) 2016 Philipp Weber
+* (C) 2019 Michael Boric
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
@@ -147,29 +148,29 @@ Test::Result test_module_get_info()
 
 std::vector<Test::Result> run_pkcs11_tests(
    const std::string& name,
-   std::vector<std::function<Test::Result()>>& fns)
+   std::vector<std::pair<std::string, std::function<Test::Result()>>>& fns)
    {
    std::vector<Test::Result> results;
 
    for(size_t i = 0; i != fns.size(); ++i)
       {
-      try
-         {
-         results.push_back(fns[ i ]());
-         }
-      catch(Botan::PKCS11::PKCS11_ReturnError& e)
-         {
-         results.push_back(Test::Result::Failure(name + " test " + std::to_string(i), e.what()));
+         try
+            {
+               results.push_back(fns[i].second());
+            }
+         catch(Botan::PKCS11::PKCS11_ReturnError& e)
+            {
+               results.push_back(Test::Result::Failure(name + " test " + fns[i].first, e.what()));
 
-         if(e.get_return_value() == Botan::PKCS11::ReturnValue::PinIncorrect)
+               if(e.get_return_value() == Botan::PKCS11::ReturnValue::PinIncorrect)
+                  {
+                     break; // Do not continue to not potentially lock the token
+                  }
+            }
+         catch(std::exception& e)
             {
-            break; // Do not continue to not potentially lock the token
+               results.push_back(Test::Result::Failure(name + " test " + fns[i].first, e.what()));
             }
-         }
-      catch(std::exception& e)
-         {
-         results.push_back(Test::Result::Failure(name + " test " + std::to_string(i), e.what()));
-         }
       }
 
    return results;
@@ -180,13 +181,12 @@ class Module_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_module_ctor,
-            test_multiple_modules,
-            test_module_get_info,
-            test_module_reload
-
+            {STRING_AND_FUNCTION(test_module_ctor)},
+            {STRING_AND_FUNCTION(test_multiple_modules)},
+            {STRING_AND_FUNCTION(test_module_get_info)},
+            {STRING_AND_FUNCTION(test_module_reload)}
             };
 
          return run_pkcs11_tests("Module", fns);
@@ -318,16 +318,16 @@ class Slot_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_slot_get_available_slots,
-            test_slot_ctor,
-            test_get_slot_info,
-            test_slot_invalid_id,
-            test_get_token_info,
-            test_get_mechanism_list,
-            test_get_mechanisms_info
-            };
+         {STRING_AND_FUNCTION(test_slot_get_available_slots)},
+         {STRING_AND_FUNCTION(test_slot_ctor)},
+         {STRING_AND_FUNCTION(test_get_slot_info)},
+         {STRING_AND_FUNCTION(test_slot_invalid_id)},
+         {STRING_AND_FUNCTION(test_get_token_info)},
+         {STRING_AND_FUNCTION(test_get_mechanism_list)},
+         {STRING_AND_FUNCTION(test_get_mechanisms_info)}
+         };
 
          return run_pkcs11_tests("Slot", fns);
          }
@@ -453,13 +453,13 @@ class Session_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_session_ctor,
-            test_session_ctor_invalid_slot,
-            test_session_release,
-            test_session_login_logout,
-            test_session_info
+            {STRING_AND_FUNCTION(test_session_ctor)},
+            {STRING_AND_FUNCTION(test_session_ctor_invalid_slot)},
+            {STRING_AND_FUNCTION(test_session_release)},
+            {STRING_AND_FUNCTION(test_session_login_logout)},
+            {STRING_AND_FUNCTION(test_session_info)}
             };
 
          return run_pkcs11_tests("Session", fns);
@@ -649,14 +649,14 @@ class Object_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_attribute_container
+            {STRING_AND_FUNCTION(test_attribute_container)}
 #if defined(BOTAN_HAS_ASN1)
-            , test_create_destroy_data_object
-            , test_get_set_attribute_values
-            , test_object_finder
-            , test_object_copy
+            , {STRING_AND_FUNCTION(test_create_destroy_data_object)}
+            , {STRING_AND_FUNCTION(test_get_set_attribute_values)}
+            , {STRING_AND_FUNCTION(test_object_finder)}
+            , {STRING_AND_FUNCTION(test_object_copy)}
 #endif
             };
 
@@ -910,15 +910,15 @@ class PKCS11_RSA_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_rsa_privkey_import,
-            test_rsa_pubkey_import,
-            test_rsa_privkey_export,
-            test_rsa_generate_private_key,
-            test_rsa_generate_key_pair,
-            test_rsa_encrypt_decrypt,
-            test_rsa_sign_verify
+            {STRING_AND_FUNCTION(test_rsa_privkey_import)},
+            {STRING_AND_FUNCTION(test_rsa_pubkey_import)},
+            {STRING_AND_FUNCTION(test_rsa_privkey_export)},
+            {STRING_AND_FUNCTION(test_rsa_generate_private_key)},
+            {STRING_AND_FUNCTION(test_rsa_generate_key_pair)},
+            {STRING_AND_FUNCTION(test_rsa_encrypt_decrypt)},
+            {STRING_AND_FUNCTION(test_rsa_sign_verify)}
             };
 
          return run_pkcs11_tests("PKCS11 RSA", fns);
@@ -1081,93 +1081,119 @@ Test::Result test_ecdsa_generate_private_key()
    return result;
    }
 
-PKCS11_ECDSA_KeyPair generate_ecdsa_keypair(const TestSession& test_session)
-   {
-   EC_PublicKeyGenerationProperties pub_props(EC_Group("secp256r1").DER_encode(
-            EC_Group_Encoding::EC_DOMPAR_ENC_OID));
-   pub_props.set_label("BOTAN_TEST_ECDSA_PUB_KEY");
-   pub_props.set_token(true);
-   pub_props.set_verify(true);
-   pub_props.set_private(false);
-   pub_props.set_modifiable(true);
-
-   EC_PrivateKeyGenerationProperties priv_props;
-   priv_props.set_label("BOTAN_TEST_ECDSA_PRIV_KEY");
-   priv_props.set_token(true);
-   priv_props.set_private(true);
-   priv_props.set_sensitive(true);
-   priv_props.set_extractable(false);
-   priv_props.set_sign(true);
-   priv_props.set_modifiable(true);
-
-   return PKCS11::generate_ecdsa_keypair(test_session.session(), pub_props, priv_props);
+PKCS11_ECDSA_KeyPair generate_ecdsa_keypair(const TestSession& test_session, const std::string& curve, EC_Group_Encoding ec_dompar_enc)
+   {
+    EC_PublicKeyGenerationProperties pub_props(EC_Group(curve).DER_encode(ec_dompar_enc));
+    pub_props.set_label("BOTAN_TEST_ECDSA_PUB_KEY");
+    pub_props.set_token(true);
+    pub_props.set_verify(true);
+    pub_props.set_private(false);
+    pub_props.set_modifiable(true);
+
+    EC_PrivateKeyGenerationProperties priv_props;
+    priv_props.set_label("BOTAN_TEST_ECDSA_PRIV_KEY");
+    priv_props.set_token(true);
+    priv_props.set_private(true);
+    priv_props.set_sensitive(true);
+    priv_props.set_extractable(false);
+    priv_props.set_sign(true);
+    priv_props.set_modifiable(true);
+
+    return PKCS11::generate_ecdsa_keypair(test_session.session(), pub_props, priv_props);
    }
 
 Test::Result test_ecdsa_generate_keypair()
    {
    Test::Result result("PKCS11 generate ECDSA key pair");
    TestSession test_session(true);
+   std::vector<std::string> curves;
 
-   PKCS11_ECDSA_KeyPair keypair = generate_ecdsa_keypair(test_session);
-   result.test_success("ECDSA key pair generation was successful");
+   curves.push_back("secp256r1");
+   curves.push_back("brainpool512r1");
 
-   keypair.first.destroy();
-   keypair.second.destroy();
+   for(auto &curve : curves)
+       {
+       PKCS11_ECDSA_KeyPair keypair = generate_ecdsa_keypair(test_session, curve, EC_DOMPAR_ENC_OID);
+
+       keypair.first.destroy();
+       keypair.second.destroy();
+       }
+   result.test_success("ECDSA key pair generation was successful");
 
    return result;
    }
 
-Test::Result test_ecdsa_sign_verify()
-   {
-   Test::Result result("PKCS11 ECDSA sign and verify");
-   TestSession test_session(true);
+Test::Result test_ecdsa_sign_verify_core(EC_Group_Encoding ec_dompar_enc, std::string test_name)
+    {
+    Test::Result result(test_name);
+    TestSession test_session(true);
+    std::vector<std::string> curves;
 
-   // generate key pair
-   PKCS11_ECDSA_KeyPair keypair = generate_ecdsa_keypair(test_session);
+    curves.push_back("secp256r1");
+    curves.push_back("brainpool512r1");
 
-   std::vector<uint8_t> plaintext(20, 0x01);
+    for(auto &curve : curves)
+        {
+        // generate key pair
+        PKCS11_ECDSA_KeyPair keypair = generate_ecdsa_keypair(test_session, curve, ec_dompar_enc);
 
-   auto sign_and_verify = [ &keypair, &plaintext, &result ](const std::string& emsa)
-      {
-      Botan::PK_Signer signer(keypair.second, Test::rng(), emsa, Botan::IEEE_1363);
-      auto signature = signer.sign_message(plaintext, Test::rng());
+        std::vector<uint8_t> plaintext(20, 0x01);
+
+        auto sign_and_verify = [&keypair, &plaintext, &result](const std::string& emsa)
+            {
+            Botan::PK_Signer signer(keypair.second, Test::rng(), emsa, Botan::IEEE_1363);
+            auto signature = signer.sign_message(plaintext, Test::rng());
 
-      Botan::PK_Verifier token_verifier(keypair.first, emsa, Botan::IEEE_1363);
-      bool ecdsa_ok = token_verifier.verify_message(plaintext, signature);
+            Botan::PK_Verifier token_verifier(keypair.first, emsa, Botan::IEEE_1363);
+            bool ecdsa_ok = token_verifier.verify_message(plaintext, signature);
 
-      result.test_eq("ECDSA PKCS11 sign and verify: " + emsa, ecdsa_ok, true);
+            result.test_eq("ECDSA PKCS11 sign and verify: " + emsa, ecdsa_ok, true);
 
-// test against software implementation if available
+            // test against software implementation if available
 #if defined (BOTAN_HAS_EMSA_RAW)
-      Botan::PK_Verifier soft_verifier(keypair.first, emsa, Botan::IEEE_1363);
-      bool soft_ecdsa_ok = soft_verifier.verify_message(plaintext, signature);
+            Botan::PK_Verifier soft_verifier(keypair.first, emsa, Botan::IEEE_1363);
+            bool soft_ecdsa_ok = soft_verifier.verify_message(plaintext, signature);
 
-      result.test_eq("ECDSA PKCS11 verify (in software): " + emsa, soft_ecdsa_ok, true);
+            result.test_eq("ECDSA PKCS11 verify (in software): " + emsa, soft_ecdsa_ok, true);
 #endif
-      };
+            };
 
-   sign_and_verify("Raw");   // SoftHSMv2 until now only supports "Raw"
+        sign_and_verify("Raw");   // SoftHSMv2 until now only supports "Raw"
 
-   keypair.first.destroy();
-   keypair.second.destroy();
+        keypair.first.destroy();
+        keypair.second.destroy();
+        }
 
-   return result;
-   }
+    return result;
+    }
+
+Test::Result test_ecdsa_sign_verify()
+    {
+        // pass the curve OID to the PKCS#11 library
+        return test_ecdsa_sign_verify_core(EC_DOMPAR_ENC_OID, "PKCS11 ECDSA sign and verify");
+    }
+
+Test::Result test_ecdsa_curve_import()
+    {
+        // pass the curve parameters to the PKCS#11 library and perform sign/verify to test them
+        return test_ecdsa_sign_verify_core(EC_DOMPAR_ENC_EXPLICIT, "PKCS11 ECDSA sign and verify with imported curve");
+    }
 
 class PKCS11_ECDSA_Tests final : public Test
    {
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_ecdsa_privkey_import,
-            test_ecdsa_privkey_export,
-            test_ecdsa_pubkey_import,
-            test_ecdsa_pubkey_export,
-            test_ecdsa_generate_private_key,
-            test_ecdsa_generate_keypair,
-            test_ecdsa_sign_verify
+            {STRING_AND_FUNCTION(test_ecdsa_privkey_import)},
+            {STRING_AND_FUNCTION(test_ecdsa_privkey_export)},
+            {STRING_AND_FUNCTION(test_ecdsa_pubkey_import)},
+            {STRING_AND_FUNCTION(test_ecdsa_pubkey_export)},
+            {STRING_AND_FUNCTION(test_ecdsa_generate_private_key)},
+            {STRING_AND_FUNCTION(test_ecdsa_generate_keypair)},
+            {STRING_AND_FUNCTION(test_ecdsa_sign_verify)},
+            {STRING_AND_FUNCTION(test_ecdsa_curve_import)}
             };
 
          return run_pkcs11_tests("PKCS11 ECDSA", fns);
@@ -1390,15 +1416,15 @@ class PKCS11_ECDH_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_ecdh_privkey_import,
-            test_ecdh_privkey_export,
-            test_ecdh_pubkey_import,
-            test_ecdh_pubkey_export,
-            test_ecdh_generate_private_key,
-            test_ecdh_generate_keypair,
-            test_ecdh_derive
+            {STRING_AND_FUNCTION(test_ecdh_privkey_import)},
+            {STRING_AND_FUNCTION(test_ecdh_privkey_export)},
+            {STRING_AND_FUNCTION(test_ecdh_pubkey_import)},
+            {STRING_AND_FUNCTION(test_ecdh_pubkey_export)},
+            {STRING_AND_FUNCTION(test_ecdh_generate_private_key)},
+            {STRING_AND_FUNCTION(test_ecdh_generate_keypair)},
+            {STRING_AND_FUNCTION(test_ecdh_derive)}
             };
 
          return run_pkcs11_tests("PKCS11 ECDH", fns);
@@ -1479,12 +1505,12 @@ class PKCS11_RNG_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_rng_generate_random
-            , test_rng_add_entropy
+            {STRING_AND_FUNCTION(test_rng_generate_random)}
+            , {STRING_AND_FUNCTION(test_rng_add_entropy)}
 #if defined(BOTAN_HAS_HMAC_DRBG )&& defined(BOTAN_HAS_SHA2_64)
-            , test_pkcs11_hmac_drbg
+            , {STRING_AND_FUNCTION(test_pkcs11_hmac_drbg)}
 #endif
             };
 
@@ -1566,12 +1592,12 @@ class PKCS11_Token_Management_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_set_pin,
-            test_initialize,
-            test_change_pin,
-            test_change_so_pin
+            {STRING_AND_FUNCTION(test_set_pin)},
+            {STRING_AND_FUNCTION(test_initialize)},
+            {STRING_AND_FUNCTION(test_change_pin)},
+            {STRING_AND_FUNCTION(test_change_so_pin)}
             };
 
          return run_pkcs11_tests("PKCS11 token management", fns);
@@ -1614,9 +1640,9 @@ class PKCS11_X509_Tests final : public Test
    public:
       std::vector<Test::Result> run() override
          {
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_x509_import
+            {STRING_AND_FUNCTION(test_x509_import)}
             };
 
          return run_pkcs11_tests("PKCS11 X509", fns);
diff --git a/src/tests/test_pkcs11_low_level.cpp b/src/tests/test_pkcs11_low_level.cpp
index 8c1a8d6ab..1b033433f 100644
--- a/src/tests/test_pkcs11_low_level.cpp
+++ b/src/tests/test_pkcs11_low_level.cpp
@@ -1,6 +1,7 @@
 /*
 * (C) 2016 Daniel Neus
 * (C) 2016 Philipp Weber
+* (C) 2019 Michael Boric
 *
 * Botan is released under the Simplified BSD License (see license.txt)
 */
@@ -795,42 +796,42 @@ class LowLevelTests final : public Test
          {
          std::vector<Test::Result> results;
 
-         std::vector<std::function<Test::Result()>> fns =
+         std::vector<std::pair<std::string, std::function<Test::Result()>>> fns =
             {
-            test_c_get_function_list
-            , test_low_level_ctor
-            , test_initialize_finalize
-            , test_c_get_info
-            , test_c_get_slot_list
-            , test_c_get_slot_info
-            , test_c_get_token_info
-            , test_c_wait_for_slot_event
-            , test_c_get_mechanism_list
-            , test_c_get_mechanism_info
-            , test_open_close_session
-            , test_c_close_all_sessions
-            , test_c_get_session_info
-            , test_c_init_token
-            , test_c_login_logout_security_officier /* only possible if token is initialized */
-            , test_c_init_pin
-            , test_c_login_logout_user /* only possible if token is initialized and user pin is set */
-            , test_c_set_pin
-            , test_c_create_object_c_destroy_object
-            , test_c_get_object_size
-            , test_c_get_attribute_value
-            , test_c_set_attribute_value
-            , test_c_copy_object
+            {STRING_AND_FUNCTION(test_c_get_function_list)}
+            , {STRING_AND_FUNCTION(test_low_level_ctor)}
+            , {STRING_AND_FUNCTION(test_initialize_finalize)}
+            , {STRING_AND_FUNCTION(test_c_get_info)}
+            , {STRING_AND_FUNCTION(test_c_get_slot_list)}
+            , {STRING_AND_FUNCTION(test_c_get_slot_info)}
+            , {STRING_AND_FUNCTION(test_c_get_token_info)}
+            , {STRING_AND_FUNCTION(test_c_wait_for_slot_event)}
+            , {STRING_AND_FUNCTION(test_c_get_mechanism_list)}
+            , {STRING_AND_FUNCTION(test_c_get_mechanism_info)}
+            , {STRING_AND_FUNCTION(test_open_close_session)}
+            , {STRING_AND_FUNCTION(test_c_close_all_sessions)}
+            , {STRING_AND_FUNCTION(test_c_get_session_info)}
+            , {STRING_AND_FUNCTION(test_c_init_token)}
+            , {STRING_AND_FUNCTION(test_c_login_logout_security_officier)} /* only possible if token is initialized */
+            , {STRING_AND_FUNCTION(test_c_init_pin)}
+            , {STRING_AND_FUNCTION(test_c_login_logout_user)} /* only possible if token is initialized and user pin is set */
+            , {STRING_AND_FUNCTION(test_c_set_pin)}
+            , {STRING_AND_FUNCTION(test_c_create_object_c_destroy_object)}
+            , {STRING_AND_FUNCTION(test_c_get_object_size)}
+            , {STRING_AND_FUNCTION(test_c_get_attribute_value)}
+            , {STRING_AND_FUNCTION(test_c_set_attribute_value)}
+            , {STRING_AND_FUNCTION(test_c_copy_object)}
             };
 
          for(size_t i = 0; i != fns.size(); ++i)
             {
             try
                {
-               results.push_back(fns[ i ]());
+               results.push_back(fns[i].second());
                }
             catch(PKCS11_ReturnError& e)
                {
-               results.push_back(Test::Result::Failure("PKCS11 low level test " + std::to_string(i), e.what()));
+               results.push_back(Test::Result::Failure("PKCS11 low level test " + fns[i].first, e.what()));
 
                if(e.get_return_value() == ReturnValue::PinIncorrect)
                   {
@@ -839,7 +840,7 @@ class LowLevelTests final : public Test
                }
             catch(std::exception& e)
                {
-               results.push_back(Test::Result::Failure("PKCS11 low level test " + std::to_string(i), e.what()));
+               results.push_back(Test::Result::Failure("PKCS11 low level test " + fns[i].first, e.what()));
                }
             }