Repository: randombit/botan
Pull request #1913 (2019-04-26 13:57:11)

Fix some build/test problems with minimized builds
Various corner case build scenarios where the tests would fail


Patch:
diff --git a/doc/release_process.rst b/doc/release_process.rst
index 37121b74f..507e9e047 100644
--- a/doc/release_process.rst
+++ b/doc/release_process.rst
@@ -25,6 +25,7 @@ constraints.  So before release, do a complete build/test cycle:
  - Native compile on FreeBSD x86-64
  - Native compile on at least one unusual platform (AIX, NetBSD, ...)
  - Build the website content to detect any Doxygen problems
+ - Test many build configurations (using `src/scripts/test_all_configs.py`)
 
 Pre Release Checks
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/src/cli/speed.cpp b/src/cli/speed.cpp
index 84d8d5679..2f4563bb3 100644
--- a/src/cli/speed.cpp
+++ b/src/cli/speed.cpp
@@ -1425,7 +1425,6 @@ class Speed final : public Command
          {
          Botan::secure_vector<Botan::word> ws;
 
-#if defined(BOTAN_HAS_NIST_PRIME_REDUCERS_W32)
          std::unique_ptr<Timer> p192_timer = make_timer("P-192 redc");
          Botan::BigInt r192(rng(), 192*2 - 1);
          while(p192_timer->under(runtime))
@@ -1465,7 +1464,6 @@ class Speed final : public Command
             r384 += 1;
             }
          record_result(p384_timer);
-#endif
 
          std::unique_ptr<Timer> p521_timer = make_timer("P-521 redc");
          Botan::BigInt r521(rng(), 521*2 - 1);
diff --git a/src/lib/math/numbertheory/curve_nistp.h b/src/lib/math/numbertheory/curve_nistp.h
index 710b06dec..963526d5a 100644
--- a/src/lib/math/numbertheory/curve_nistp.h
+++ b/src/lib/math/numbertheory/curve_nistp.h
@@ -23,6 +23,11 @@ namespace Botan {
 BOTAN_PUBLIC_API(2,0) const BigInt& prime_p521();
 BOTAN_PUBLIC_API(2,0) void redc_p521(BigInt& x, secure_vector<word>& ws);
 
+/*
+Previously this macro indicated if the P-{192,224,256,384} reducers
+were available. Now they are always enabled and this macro has no meaning.
+The define will be removed in a future major release.
+*/
 #define BOTAN_HAS_NIST_PRIME_REDUCERS_W32
 
 BOTAN_PUBLIC_API(2,0) const BigInt& prime_p384();
diff --git a/src/lib/math/numbertheory/nistp_redc.cpp b/src/lib/math/numbertheory/nistp_redc.cpp
index 17089fcbe..ea667f8b6 100644
--- a/src/lib/math/numbertheory/nistp_redc.cpp
+++ b/src/lib/math/numbertheory/nistp_redc.cpp
@@ -75,8 +75,6 @@ void redc_p521(BigInt& x, secure_vector<word>& ws)
    bigint_cnd_sub(needs_reduction.value(), x.mutable_data(), p521_words, p_words);
    }
 
-#if defined(BOTAN_HAS_NIST_PRIME_REDUCERS_W32)
-
 namespace {
 
 /**
@@ -586,6 +584,4 @@ void redc_p384(BigInt& x, secure_vector<word>& ws)
    bigint_cnd_add(borrow, x.mutable_data(), p384_limbs + 1, p384_mults[0], p384_limbs);
    }
 
-#endif
-
 }
diff --git a/src/lib/pk_pad/eme.cpp b/src/lib/pk_pad/eme.cpp
index 5164157f7..ffedac923 100644
--- a/src/lib/pk_pad/eme.cpp
+++ b/src/lib/pk_pad/eme.cpp
@@ -14,7 +14,7 @@
 #include <botan/oaep.h>
 #endif
 
-#if defined(BOTAN_HAS_EME_PKCS1v15)
+#if defined(BOTAN_HAS_EME_PKCS1)
 #include <botan/eme_pkcs.h>
 #endif
 
@@ -31,7 +31,7 @@ EME* get_eme(const std::string& algo_spec)
       return new EME_Raw;
 #endif
 
-#if defined(BOTAN_HAS_EME_PKCS1v15)
+#if defined(BOTAN_HAS_EME_PKCS1)
    if(algo_spec == "PKCS1v15" || algo_spec == "EME-PKCS1-v1_5")
       return new EME_PKCS1v15;
 #endif
diff --git a/src/lib/pk_pad/eme_pkcs1/info.txt b/src/lib/pk_pad/eme_pkcs1/info.txt
index 7c827cc34..772806e42 100644
--- a/src/lib/pk_pad/eme_pkcs1/info.txt
+++ b/src/lib/pk_pad/eme_pkcs1/info.txt
@@ -1,3 +1,4 @@
 <defines>
 EME_PKCS1v15 -> 20131128
+EME_PKCS1 -> 20190426
 </defines>
diff --git a/src/lib/pubkey/ec_group/curve_gfp.cpp b/src/lib/pubkey/ec_group/curve_gfp.cpp
index 61ebfda35..9957bb085 100644
--- a/src/lib/pubkey/ec_group/curve_gfp.cpp
+++ b/src/lib/pubkey/ec_group/curve_gfp.cpp
@@ -279,8 +279,6 @@ void CurveGFp_NIST::curve_sqr_words(BigInt& z, const word x[], size_t x_size,
    this->redc_mod_p(z, ws);
    }
 
-#if defined(BOTAN_HAS_NIST_PRIME_REDUCERS_W32)
-
 /**
 * The NIST P-192 curve
 */
@@ -466,8 +464,6 @@ BigInt CurveGFp_P384::invert_element(const BigInt& x, secure_vector<word>& ws) c
    return r;
    }
 
-#endif
-
 /**
 * The NIST P-521 curve
 */
@@ -555,7 +551,6 @@ BigInt CurveGFp_P521::invert_element(const BigInt& x, secure_vector<word>& ws) c
 std::shared_ptr<CurveGFp_Repr>
 CurveGFp::choose_repr(const BigInt& p, const BigInt& a, const BigInt& b)
    {
-#if defined(BOTAN_HAS_NIST_PRIME_REDUCERS_W32)
    if(p == prime_p192())
       return std::shared_ptr<CurveGFp_Repr>(new CurveGFp_P192(a, b));
    if(p == prime_p224())
@@ -564,8 +559,6 @@ CurveGFp::choose_repr(const BigInt& p, const BigInt& a, const BigInt& b)
       return std::shared_ptr<CurveGFp_Repr>(new CurveGFp_P256(a, b));
    if(p == prime_p384())
       return std::shared_ptr<CurveGFp_Repr>(new CurveGFp_P384(a, b));
-#endif
-
    if(p == prime_p521())
       return std::shared_ptr<CurveGFp_Repr>(new CurveGFp_P521(a, b));
 
diff --git a/src/lib/x509/x509cert.cpp b/src/lib/x509/x509cert.cpp
index dda15835e..de4b0ed7a 100644
--- a/src/lib/x509/x509cert.cpp
+++ b/src/lib/x509/x509cert.cpp
@@ -287,6 +287,13 @@ std::unique_ptr<X509_Certificate_Data> parse_x509_cert_body(const X509_Object& o
          }
       else
          {
+         /*
+         If a parse error or unknown algorithm is encountered, default
+         to assuming it is self signed. We have no way of being certain but
+         that is usually the default case (self-issued is rare in practice).
+         */
+         data->m_self_signed = true;
+
          try
             {
             std::unique_ptr<Public_Key> pub_key(X509::load_key(data->m_subject_public_key_bits_seq));
@@ -298,6 +305,10 @@ std::unique_ptr<X509_Certificate_Data> parse_x509_cert_body(const X509_Object& o
                {
                data->m_self_signed = true;
                }
+            else
+               {
+               data->m_self_signed = false;
+               }
             }
          catch(...)
             {
diff --git a/src/scripts/macro_checks.py b/src/scripts/macro_checks.py
new file mode 100755
index 000000000..df1a503c6
--- /dev/null
+++ b/src/scripts/macro_checks.py
@@ -0,0 +1,42 @@
+#!/usr/bin/python
+
+# (C) 2018 Jack Lloyd
+# Botan is released under the Simplified BSD License (see license.txt)
+
+# Scans all source and test files and makes sure we are not using a
+# BOTAN_HAS_xxx macro which is not actually defined anywhere.
+
+from configure import ModuleInfo, load_info_files
+import os
+import re
+import logging
+
+src_dir = 'src'
+lib_dir = os.path.join(src_dir, 'lib')
+
+info_modules = load_info_files(lib_dir, 'Modules', "info.txt", ModuleInfo)
+
+all_defines = set()
+
+for module in info_modules.values():
+    for define in module._defines:
+        all_defines.add(define)
+
+extras = ['MP_DWORD', 'VALGRIND', 'SANITIZER_UNDEFINED',
+          'ONLINE_REVOCATION_CHECKS', 'NIST_PRIME_REDUCERS_W32']
+
+for extra in extras:
+    all_defines.add(extra)
+
+macro = re.compile('BOTAN_HAS_([A-Z0-9_]+)')
+
+for dirname, subdirs, files in os.walk(src_dir):
+    for fname in files:
+        if fname.endswith('.h') or fname.endswith('.cpp'):
+            contents = open(os.path.join(dirname, fname)).read()
+
+            for m in re.finditer(macro, contents):
+
+                if m.group(1) not in all_defines:
+                    logging.error('In %s found unknown feature macro %s' % (fname, m.group(1)))
+


Test patch:
diff --git a/src/cli/timing_tests.cpp b/src/cli/timing_tests.cpp
index 63e04b32d..3f5152ec1 100644
--- a/src/cli/timing_tests.cpp
+++ b/src/cli/timing_tests.cpp
@@ -44,7 +44,7 @@
    #include <botan/system_rng.h>
 #endif
 
-#if defined(BOTAN_HAS_AUTO_SEEDED_RNG)
+#if defined(BOTAN_HAS_AUTO_SEEDING_RNG)
    #include <botan/auto_rng.h>
 #endif
 
@@ -96,7 +96,7 @@ class Timing_Test
          {
 #if defined(BOTAN_HAS_SYSTEM_RNG)
          return Botan::system_rng();
-#elif defined(BOTAN_HAS_AUTO_SEEDED_RNG)
+#elif defined(BOTAN_HAS_AUTO_SEEDING_RNG)
          static AutoSeeded_RNG static_timing_test_rng(Botan::Entropy_Sources::global_sources(), 0);
          return static_timing_test_rng;
 #else
@@ -107,7 +107,7 @@ class Timing_Test
 
    };
 
-#if defined(BOTAN_HAS_RSA) && defined(BOTAN_HAS_EME_PKCS1v15) && defined(BOTAN_HAS_EME_RAW)
+#if defined(BOTAN_HAS_RSA) && defined(BOTAN_HAS_EME_PKCS1) && defined(BOTAN_HAS_EME_RAW)
 
 class Bleichenbacker_Timing_Test final : public Timing_Test
    {
@@ -544,7 +544,7 @@ BOTAN_REGISTER_COMMAND("timing_test", Timing_Test_Command);
 
 std::unique_ptr<Timing_Test> Timing_Test_Command::lookup_timing_test(const std::string& test_type)
    {
-#if defined(BOTAN_HAS_RSA) && defined(BOTAN_HAS_EME_PKCS1v15) && defined(BOTAN_HAS_EME_RAW)
+#if defined(BOTAN_HAS_RSA) && defined(BOTAN_HAS_EME_PKCS1) && defined(BOTAN_HAS_EME_RAW)
    if(test_type == "bleichenbacher")
       {
       return std::unique_ptr<Timing_Test>(new Bleichenbacker_Timing_Test(2048));
diff --git a/src/scripts/test_all_configs.py b/src/scripts/test_all_configs.py
index efa07430a..208720c55 100755
--- a/src/scripts/test_all_configs.py
+++ b/src/scripts/test_all_configs.py
@@ -89,11 +89,10 @@ def main(args):
     parser.add_option('--jobs', default=get_concurrency(),
                       help="jobs to run (default %default)")
 
-    jobs = get_concurrency()
-
     (options, args) = parser.parse_args(args)
 
     run_tests = options.run_tests
+    jobs = int(options.jobs)
 
     configure_py = './configure.py'
     modules = get_module_list(configure_py)
diff --git a/src/tests/test_ecdsa.cpp b/src/tests/test_ecdsa.cpp
index e1819fe1a..296ae5175 100644
--- a/src/tests/test_ecdsa.cpp
+++ b/src/tests/test_ecdsa.cpp
@@ -129,7 +129,7 @@ class ECDSA_Signature_KAT_Tests final : public PK_Signature_Generation_Test
          return "EMSA1(" + hash + ")";
          }
 
-#if !defined(BOTAN_HAS_RFC6979)
+#if !defined(BOTAN_HAS_RFC6979_GENERATOR)
       Botan::RandomNumberGenerator* test_rng(const std::vector<uint8_t>& nonce) const override
          {
          // probabilistic ecdsa signature generation extracts more random than just the nonce,
diff --git a/src/tests/test_ecies.cpp b/src/tests/test_ecies.cpp
index 5dd70afd7..534a2719e 100644
--- a/src/tests/test_ecies.cpp
+++ b/src/tests/test_ecies.cpp
@@ -16,7 +16,7 @@ namespace Botan_Tests {
 
 namespace {
 
-#if defined(BOTAN_HAS_ECIES)
+#if defined(BOTAN_HAS_ECIES) && defined(BOTAN_HAS_AES) && defined(BOTAN_HAS_MODE_CBC)
 
 using Flags = Botan::ECIES_Flags;
 
diff --git a/src/tests/test_ffi.cpp b/src/tests/test_ffi.cpp
index 696a70377..f2926d777 100644
--- a/src/tests/test_ffi.cpp
+++ b/src/tests/test_ffi.cpp
@@ -87,11 +87,11 @@ class FFI_Unit_Tests final : public Test
          results.push_back(ffi_test_keywrap());
 #endif
 
-#if defined(BOTAN_HAS_HOTP)
+#if defined(BOTAN_HAS_HOTP) && defined(BOTAN_HAS_SHA1)
          results.push_back(ffi_test_hotp());
 #endif
 
-#if defined(BOTAN_HAS_TOTP)
+#if defined(BOTAN_HAS_TOTP) && defined(BOTAN_HAS_SHA1)
          results.push_back(ffi_test_totp());
 #endif
 
@@ -109,7 +109,7 @@ class FFI_Unit_Tests final : public Test
          results.push_back(ffi_test_ecdsa_cert());
 #endif
 
-#if defined(BOTAN_HAS_ECDH)
+#if defined(BOTAN_HAS_ECDH) && defined(BOTAN_HAS_KDF2) && defined(BOTAN_HAS_SHA2_32)
          results.push_back(ffi_test_ecdh(rng));
 #endif
 
@@ -122,7 +122,7 @@ class FFI_Unit_Tests final : public Test
          results.push_back(ffi_test_mceliece(rng));
 #endif
 
-#if defined(BOTAN_HAS_ELGAMAL)
+#if defined(BOTAN_HAS_ELGAMAL) && defined(BOTAN_HAS_EME_RAW)
          results.push_back(ffi_test_elgamal(rng));
 #endif
 
@@ -281,7 +281,7 @@ class FFI_Unit_Tests final : public Test
       Test::Result ffi_test_cert_validation()
          {
          Test::Result result("FFI Cert validation");
-#if defined(BOTAN_HAS_X509_CERTIFICATES)
+#if defined(BOTAN_HAS_X509_CERTIFICATES) && defined(BOTAN_HAS_RSA)
 
          botan_x509_cert_t root;
          int rc;
@@ -470,6 +470,7 @@ class FFI_Unit_Tests final : public Test
          {
          Test::Result result("FFI CBC cipher");
 
+#if defined(BOTAN_HAS_AES) && defined(BOTAN_HAS_MODE_CBC)
          botan_cipher_t cipher_encrypt, cipher_decrypt;
 
          if(TEST_FFI_OK(botan_cipher_init, (&cipher_encrypt, "AES-128/CBC/PKCS7", BOTAN_CIPHER_INIT_FLAG_ENCRYPT)))
@@ -536,6 +537,7 @@ class FFI_Unit_Tests final : public Test
 
             TEST_FFI_OK(botan_cipher_destroy, (cipher_encrypt));
             }
+#endif
 
          return result;
          }
@@ -1420,8 +1422,6 @@ class FFI_Unit_Tests final : public Test
 
       void ffi_test_pubkey_export(Test::Result& result, botan_pubkey_t pub, botan_privkey_t priv, botan_rng_t rng)
          {
-         const size_t pbkdf_iter = 1000;
-
          // export public key
          size_t pubkey_len = 0;
          TEST_FFI_RC(BOTAN_FFI_ERROR_INSUFFICIENT_BUFFER_SPACE, botan_pubkey_export, (pub, nullptr, &pubkey_len,
@@ -1477,6 +1477,9 @@ class FFI_Unit_Tests final : public Test
          TEST_FFI_OK(botan_privkey_load, (&copy, rng, privkey.data(), privkey.size(), nullptr));
          botan_privkey_destroy(copy);
 
+#if defined(BOTAN_HAS_AES) && defined(BOTAN_HAS_PKCS5_PBES2)
+         const size_t pbkdf_iter = 1000;
+
          // export private key encrypted
          privkey_len = 0;
          TEST_FFI_RC(BOTAN_FFI_ERROR_INSUFFICIENT_BUFFER_SPACE, botan_privkey_export_encrypted_pbkdf_iter, (priv, nullptr,
@@ -1511,7 +1514,7 @@ class FFI_Unit_Tests final : public Test
          const std::string pbe_hash = "SHA-512";
 #endif
 
-#if defined(BOTAN_HAS_GCM)
+#if defined(BOTAN_HAS_AEAD_GCM)
          const std::string pbe_cipher = "AES-256/GCM";
 #else
          const std::string pbe_cipher = "AES-256/CBC";
@@ -1535,6 +1538,7 @@ class FFI_Unit_Tests final : public Test
 
          TEST_FFI_OK(botan_privkey_load, (&copy, rng, privkey.data(), privkey.size(), "password"));
          botan_privkey_destroy(copy);
+#endif
 
          // calculate fingerprint
          size_t strength = 0;
@@ -1790,6 +1794,7 @@ class FFI_Unit_Tests final : public Test
                result.test_eq("algo name", std::string(namebuf), "RSA");
                }
 
+#if defined(BOTAN_HAS_EME_OAEP)
             botan_pk_op_encrypt_t encrypt;
 
             if(TEST_FFI_OK(botan_pk_op_encrypt_create, (&encrypt, loaded_pubkey, "OAEP(SHA-256)", 0)))
@@ -1825,6 +1830,7 @@ class FFI_Unit_Tests final : public Test
 
                TEST_FFI_OK(botan_pk_op_encrypt_destroy, (encrypt));
                }
+#endif
 
             TEST_FFI_OK(botan_pubkey_destroy, (loaded_pubkey));
             TEST_FFI_OK(botan_pubkey_destroy, (pub));
diff --git a/src/tests/test_pk_pad.cpp b/src/tests/test_pk_pad.cpp
index 7c92cf876..83a712b40 100644
--- a/src/tests/test_pk_pad.cpp
+++ b/src/tests/test_pk_pad.cpp
@@ -10,13 +10,13 @@
    #include <botan/emsa.h>
 #endif
 
-#if defined(BOTAN_HAS_EME_PKCS1v15)
+#if defined(BOTAN_HAS_EME_PKCS1)
    #include <botan/eme_pkcs.h>
 #endif
 
 namespace Botan_Tests {
 
-#if defined(BOTAN_HAS_EME_PKCS1v15)
+#if defined(BOTAN_HAS_EME_PKCS1)
 
 class EME_PKCS1v15_Decoding_Tests final : public Text_Based_Test
    {
diff --git a/src/tests/unit_ecc.cpp b/src/tests/unit_ecc.cpp
index 869cda277..28b2fadd1 100644
--- a/src/tests/unit_ecc.cpp
+++ b/src/tests/unit_ecc.cpp
@@ -185,13 +185,13 @@ class NIST_Curve_Reduction_Tests final : public Test
    {
    public:
       typedef std::function<void (Botan::BigInt&, Botan::secure_vector<Botan::word>&)> reducer_fn;
+
       std::vector<Test::Result> run() override
          {
          std::vector<Test::Result> results;
 
          // Using lambdas here to avoid strange UbSan warning (#1370)
 
-#if defined(BOTAN_HAS_NIST_PRIME_REDUCERS_W32)
          results.push_back(random_redc_test("P-384", Botan::prime_p384(),
                               [](Botan::BigInt& p, Botan::secure_vector<Botan::word>& ws) -> void
                                  {
@@ -212,7 +212,6 @@ class NIST_Curve_Reduction_Tests final : public Test
                                  {
                                  Botan::redc_p192(p, ws);
                                  }));
-#endif
          results.push_back(random_redc_test("P-521", Botan::prime_p521(),
                               [](Botan::BigInt& p, Botan::secure_vector<Botan::word>& ws) -> void
                                  {
diff --git a/src/tests/unit_tls.cpp b/src/tests/unit_tls.cpp
index 28b09324f..fa23b5842 100644
--- a/src/tests/unit_tls.cpp
+++ b/src/tests/unit_tls.cpp
@@ -906,7 +906,7 @@ class TLS_Unit_Tests final : public Test
 #if defined(BOTAN_HAS_CAMELLIA) && defined(BOTAN_HAS_TLS_CBC)
          test_modern_versions("Camellia-256 SHA-2", results, *client_ses, *server_ses, *creds, "RSA", "Camellia-256", "SHA-384 SHA-256");
 #endif
-#if defined(BOTAN_HAS_CAMELLIA) && defined(BOTAN_HAS_GCM)
+#if defined(BOTAN_HAS_CAMELLIA) && defined(BOTAN_HAS_AEAD_GCM)
          test_modern_versions("Camellia-128/GCM ECDH", results, *client_ses, *server_ses, *creds, "ECDH", "Camellia-128/GCM", "AEAD");
 #endif
 
@@ -975,7 +975,7 @@ class TLS_Unit_Tests final : public Test
 
          test_modern_versions("AES-128/GCM PSK", results, *client_ses, *server_ses, *creds, "PSK", "AES-128/GCM");
 
-#if defined(BOTAN_HAS_CCM)
+#if defined(BOTAN_HAS_AEAD_CCM)
          test_modern_versions("AES-128/CCM PSK", results, *client_ses, *server_ses, *creds, "PSK", "AES-128/CCM");
          test_modern_versions("AES-128/CCM-8 PSK", results, *client_ses, *server_ses, *creds, "PSK", "AES-128/CCM(8)");
 #endif