Repository: conan-io/conan-center-index

Issue:
[package] openssl/1.1.1g: Cannot make HTTPS requests using Boost.Beast
<!-- 
  Please don't forget to update the issue title.
  Include all applicable information to help us reproduce your problem.
-->

### Package and Environment Details (include every applicable attribute)
  * Package Name/Version: **openssl/1.1.1g** and **boost/1.74**
  * Operating System+version: **Linux Ubuntu 20.04 (docker)**
  * Compiler+version: **GCC 10.0.1**
  * Docker image: **ubuntu/20.04**
  * Conan version: **Conan version 1.28.1**
  * Python version: **Python 3.8.2**


### Conan profile (output of `conan profile show default` or `conan profile show <profile>` if custom profile is in use)
```
Configuration for profile gcc10:

[settings]
arch=x86_64
arch_build=x86_64
build_type=Release
compiler=gcc
compiler.libcxx=libstdc++11
compiler.version=10
os=Linux
os_build=Linux
[options]
[build_requires]
[env]
CC=/usr/bin/gcc-10
CXX=/usr/bin/g++-10
```


### Steps to reproduce (Include if Applicable)

Have a conan file with this content:

```
[requires]
boost/1.74.0
openssl/1.1.1g

[generators]
cmake
```

And such CMake file:

```cmake
cmake_minimum_required(VERSION 3.17)
project(beast-test)

include("${PROJECT_BINARY_DIR}/conanbuildinfo.cmake")
conan_basic_setup(TARGETS)

set(CMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY ON)
set(CMAKE_FIND_USE_CMAKE_SYSTEM_PATH OFF)
set(CMAKE_FIND_PACKAGE_PREFER_CONFIG ON)
set(CMAKE_FIND_USE_CMAKE_PATH OFF)
set(CMAKE_FIND_USE_CMAKE_ENVIRONMENT_PATH OFF)

add_executable(http-beast-ssl main.cpp)
target_link_libraries(http-beast-ssl PRIVATE CONAN_PKG::boost CONAN_PKG::openssl)
```

The main file looks like this:
```c++
#include <boost/asio/ssl.hpp>
#include <boost/lexical_cast.hpp>
#include <openssl/cryptoerr.h>
#include <boost/beast/core.hpp>
#include <boost/beast/http.hpp>
#include <boost/beast/ssl.hpp>
#include <boost/beast/version.hpp>
#include <boost/asio/connect.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <boost/asio/ssl/error.hpp>
#include <boost/asio/ssl/stream.hpp>
#include <boost/beast/version.hpp>
#include <openssl/opensslv.h>
#include <cstdlib>
#include <iostream>
#include <string>

namespace beast = boost::beast; // from <boost/beast.hpp>
namespace http = beast::http;   // from <boost/beast/http.hpp>
namespace net = boost::asio;    // from <boost/asio.hpp>
namespace ssl = net::ssl;       // from <boost/asio/ssl.hpp>
using tcp = net::ip::tcp;       // from <boost/asio/ip/tcp.hpp>

// Performs an HTTP GET and prints the response
int main(int argc, char** argv)
{
    std::cout << OPENSSL_VERSION_TEXT << std::endl;
    try
    {
        // Check command line arguments.
        if(argc != 4 && argc != 5)
        {
            std::cerr <<
                "Usage: http-client-sync-ssl <host> <port> <target> [<HTTP version: 1.0 or 1.1(default)>]\n" <<
                "Example:\n" <<
                "    http-client-sync-ssl www.example.com 443 /\n" <<
                "    http-client-sync-ssl www.example.com 443 / 1.0\n";
            return EXIT_FAILURE;
        }
        auto const host = argv[1];
        auto const port = argv[2];
        auto const target = argv[3];
        int version = argc == 5 && !std::strcmp("1.0", argv[4]) ? 10 : 11;

        // The io_context is required for all I/O
        net::io_context ioc;

        // The SSL context is required, and holds certificates
        ssl::context ctx(ssl::context::tlsv12_client);

        // This holds the root certificate used for verification
		ctx.set_default_verify_paths();

        // Verify the remote server's certificate
        ctx.set_verify_mode(ssl::verify_peer);

            // These objects perform our I/O
        tcp::resolver resolver(ioc);
        beast::ssl_stream<beast::tcp_stream> stream(ioc, ctx);

        // Set SNI Hostname (many hosts need this to handshake successfully)
        if(! SSL_set_tlsext_host_name(stream.native_handle(), host))
        {
            beast::error_code ec{static_cast<int>(::ERR_get_error()), net::error::get_ssl_category()};
            throw beast::system_error{ec};
        }

        // Look up the domain name
        auto const results = resolver.resolve(host, port);

        // Make the connection on the IP address we get from a lookup
        beast::get_lowest_layer(stream).connect(results);

        // Perform the SSL handshake
        stream.handshake(ssl::stream_base::client);

        // Set up an HTTP GET request message
        http::request<http::string_body> req{http::verb::get, target, version};
        req.set(http::field::host, host);
        req.set(http::field::user_agent, BOOST_BEAST_VERSION_STRING);

        // Send the HTTP request to the remote host
        http::write(stream, req);

        // This buffer is used for reading and must be persisted
        beast::flat_buffer buffer;

        // Declare a container to hold the response
        http::response<http::dynamic_body> res;

        // Receive the HTTP response
        http::read(stream, buffer, res);

        // Write the message to standard out
        std::cout << res << std::endl;

        // Gracefully close the stream
        beast::error_code ec;
        stream.shutdown(ec);
        if(ec == net::error::eof)
        {
            // Rationale:
            // http://stackoverflow.com/questions/25587403/boost-asio-ssl-async-shutdown-always-finishes-with-an-error
            ec = {};
        }
        if(ec)
            throw beast::system_error{ec};

        // If we get here then the connection is closed gracefully
    }
    catch(boost::system::system_error const& e)
    {
        auto const& error = e.code();
        std::string err = error.message();
        if (error.category() == boost::asio::error::get_ssl_category()) {
            err = std::string(" (")
                +boost::lexical_cast<std::string>(ERR_GET_LIB(error.value()))+","
                +boost::lexical_cast<std::string>(ERR_GET_FUNC(error.value()))+","
                +boost::lexical_cast<std::string>(ERR_GET_REASON(error.value()))+") ";
    
            //ERR_PACK /* crypto/err/err.h */
            char buf[128];
            ::ERR_error_string_n(error.value(), buf, sizeof(buf));
            err += buf;
            std::cerr << err << std::endl;
        }
        return EXIT_FAILURE;
    }
    return EXIT_SUCCESS;
    
}
```

Running this:

    ./bin/http-beast-ssl example.com 443 /

Output this error:

```
OpenSSL 1.1.1g  21 Apr 2020
(20,367,134) error:1416F086:SSL routines:tls_process_server_certificate:certificate verify failed
```

### Important notice

This error **don't happen** when using system's openssl package. The issue I'm reporting here is the results from the findings of boostorg/beast#2119

I'm reporting this issue here because it should either:

 - Work out of the box, just like using system's openssl
 - Have a documented way to make it work

Unfortunately, it seem neither of these points are satisfied.

For it to work, it seems openssl should be allowed to read from `/etc/ssl/certs`, which I suspect that openssl is not configured that way.


Discussion:
To confirm that this is a Conan problem, the very same code using vcpkg will work as intended.
Not a fix, but I got it the example to work on my Fedora machine by passing a path to the ca certificate store:
```
SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt bin/http-beast-ssl example.com 443 /
```

I tried passing some `-o openssl:openssldir=...` with paths to my system, but it didn't work.

This begs the question whether we need to create a package containing the root certificates?
Fedora uses the certificates from Mozilla nss:
See https://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib/ckfw/builtins/certdata.txt for the certs,
and see https://hg.mozilla.org/releases/mozilla-release/raw-file/default/security/nss/lib/ckfw/builtins/nssckbi.h for the version.
> This begs the question whether we need to create a package containing the root certificates?

the security risk of this a huge liability to JFrog...

Since the environment variable seems be a standard for openssl, perhaps there's no bug?
@prince-chrismc This is hardly a plaster. OpenSSL should just work, and libraries using it should also just work as advertised, without special treatment on the app because in the build it uses conan.
@prince-chrismc 
Does building the example from OP work for you?
By work, I mean showing the html of example.com.

About liability, I am no lawyer but adding a matching license + big disclaimer should do.
I'm cc'ing some stakeholders  @uilianries @danimtb @jgsogo @SSE4 @memsharded @lasote 
@gracicot Je m'excuse if I came across as rude or dismissive, I should have ajoute un thinking emoji to better convey what I meant, let me try again!

💭 perhaps this is just a missing option from the recipe?

⛵ not everyone has the same use case, my applications tend to run in private clouds with no public access to perform CRL or OSCP checks... This is probably sparing me nightmare-ish 🐛 reports.

Lastly, we are hiring! If you want to send me your CV, my team gets all the bleeding edge IP work (in C++ hehe). If you are up for a challenge that is.... 😉 

@madebr I never tried it, I am too familiar with this issue, I believed it the second I saw the issue.

For me, based on my experience, the big impediment is that one ca bundle is never enough to cover all the Root CA providers. The OS already does this for us... compare the [wiki list ](https://en.wikipedia.org/wiki/Certificate_authority#Providers) to the ones you have installed. How many are missing? Have you ever found one? 

What scares me the most, there's an issue in CI, some bad bundle gets published? Then what? How do we recall that?

You're solution is 100% viable and it can for sure work. My personal thought would be to code the path in like libressl and avoid the hassle.
@prince-chrismc no problem, I didn't wanted to be rude or dismissive either, but rather emphasize on what's the perspective of a user with no prior knowledge of the specifics of the package manager. But still, whether this is a bug or not is still worth asking.

For example, if a user try a package manager and the app doesn't work but works on another package manager, the user will instantly blame the package manager, even though it simply tries to be stricter with how the app is interfacing with the rest of the system.
💯 agree
I believe we should check what vcpkg is doing differently from us with openssl. I think openssl should work the same way without the application providing the root to the certificates and using the defaults picked by openssl.

The idea of creating a package with the certificates sounds a bit weird to me and as a user, I wouldn't be comfortable relying on a bundle of certificates provided somehow artificially by Conan.
The problem seems to be that the default certificate directory on Conan's OpenSSL is set to a location inside the conan package instead of being system-wide. To test this, I've built and run the following test program in a few different ways:

``` c++
#include <cstdio>
#include <openssl/x509.h>

int main(int argc, char *argv[]) {
  const char *dir;
  dir = getenv(X509_get_default_cert_dir_env());
  if (!dir)
    dir = X509_get_default_cert_dir();
  puts(dir);
}
```

Depending on the OpenSSL that I use, I get the following paths on an Ubuntu 20.04 system:

- System OpenSSL (v1.1.1f): `/usr/lib/ssl/certs` (this is a symbolic link to `/etc/ssl/certs`)
- Vcpkg: `/etc/ssl/certs`
- Conan: `/home/conan/w/cci_PR-4460/.conan/data/openssl/1.1.1i/_/_/package/23b828d52c0630e6b0b96d2945419feb7843c4f8/res/certs` (which doesn't exist)

Looking at Vcpkg, they use an `--openssldir=/etc/ssl` option for the [Unix version](https://github.com/microsoft/vcpkg/blob/master/ports/openssl/unix/CMakeLists.txt#L171) which I think explains why the default SSL directory is system-wide.

Using the Conan package, passing the option `-o openssl:openssldir=/etc/ssl` on my system fixes the problem.

Note that the default certificate path on CentOS/RHEL is `/etc/pki/tls/certs`, but at least on my test machines (CentOS 7 and CentOS 8 Stream), `/etc/ssl/certs` is a symbolic link to `/etc/pki/tls/certs`.

Would it make sense to change the Conan OpenSSL recipe to set `openssldir`  to `/etc/ssl` by default on Linux?
@rdeterre This has been sitting in my inbox for so long I missed your comment.

Thank you so much for taking the time to investigate this issue, I really appreciate you confirming what was mentioned above.

# ❤️ 

> Would it make sense to change the Conan OpenSSL recipe to set openssldir to /etc/ssl by default on Linux?

💯 I think it would be amazing to fix that. Do you have the time to open the pull request? If not I'll see to squeeze this in this week.

Should be a small change https://github.com/conan-io/conan-center-index/blob/0cce2f0ebff06870fc288fc99eaa746dad25df89/recipes/boost/all/conanfile.py#L212-L213 not ^^ + setting the value

It seems our current behavoir matches with windows 
https://github.com/microsoft/vcpkg/blob/e25b620cf90d9bb311f91b03916b75d8798afd44/ports/openssl/windows/portfile.cmake#L66
No problem, I'm be happy to make a pull request :)

Given that the issue is not only with Boost but with any package that uses OpenSSL, I'm thinking it would make sense to change the default value of "openssldir" to "/etc/ssl" directly there. I'm not sure if I'm doing this the right way but I have a patch that works on my systems. I'll push this right away.

Resolving patch:
diff --git a/recipes/openssl/1.x.x/conanfile.py b/recipes/openssl/1.x.x/conanfile.py
--- a/recipes/openssl/1.x.x/conanfile.py
+++ b/recipes/openssl/1.x.x/conanfile.py
@@ -436,9 +436,14 @@ def _get_env_build(self):
             self._env_build = AutoToolsBuildEnvironment(self)
         return self._env_build
 
+    def _get_default_openssl_dir(self):
+        if self.settings.os == "Linux" and self._full_version >= "1.1.0":
+            return "/etc/ssl"
+        return os.path.join(self.package_folder, "res")
+
     @property
     def _configure_args(self):
-        openssldir = self.options.openssldir if self.options.openssldir else os.path.join(self.package_folder, "res")
+        openssldir = self.options.openssldir or self._get_default_openssl_dir()
         prefix = tools.unix_path(self.package_folder) if self._win_bash else self.package_folder
         openssldir = tools.unix_path(openssldir) if self._win_bash else openssldir
         args = [


Test patch: