CVE-2020-9432 (C, published 2020-02-27T23:15Z)
Description: value: openssl_x509_check_host in lua-openssl 0.7.7-1 mishandles X.509 certificate validation because it uses lua_pushboolean for certain non-boolean return values.
Severity: MEDIUM (CVSS v3 base score: 9.1)
Weakness: CWE-295 \u2014 Improper Certificate Validation
The product does not validate, or incorrectly validates, a certificate.
Repository: https://github.com/zhaozg/lua-openssl
Files changed:
src/x509.c
Commit message: fix security: GSL-2020-026


Vulnerable code:
  if (lua_isstring(L, 2))
  {
    const char *hostname = lua_tostring(L, 2);
    lua_pushboolean(L, X509_check_host(cert, hostname, strlen(hostname), 0, NULL));
  }
  else
  {
    lua_pushboolean(L, 0);
  }
  return 1;
  if (lua_isstring(L, 2))
  {
    const char *email = lua_tostring(L, 2);
    lua_pushboolean(L, X509_check_email(cert, email, strlen(email), 0));
  }
  else
  {
    lua_pushboolean(L, 0);
  }
  return 1;
static LUA_FUNCTION(openssl_x509_check_ip_asc)
  if (lua_isstring(L, 2))
  {
    const char *ip_asc = lua_tostring(L, 2);
    lua_pushboolean(L, X509_check_ip_asc(cert, ip_asc, 0));
  }
  else
  {
    lua_pushboolean(L, 0);
  }
  return 1;
  {"check_ip_asc", openssl_x509_check_ip_asc},

Fixed code:
/***
The functions return 1 for a successful match, 0 for a failed match and -1 for
an internal error: typically a memory allocation failure or an ASN.1 decoding
error.

All functions can also return -2 if the input is malformed. For example,
X509_check_host() returns -2 if the provided name contains embedded NULs.
*/
static int openssl_push_check_result(lua_State *L, int ret, const char* name)
{
  switch (ret)
  {
  case 1:
    lua_pushboolean(L, 1);
    if (name)
    {
      lua_pushstring(L, name);
      ret = 2;
    }
    break;
  case 0:
    lua_pushboolean(L, 0);
    ret = 1;
    break;
  case -1:
    lua_pushnil(L);
    lua_pushliteral(L, "internal");
    ret = 2;
  case -2:
    lua_pushnil(L);
    lua_pushliteral(L, "malformed");
    ret = 2;
  default:
    lua_pushnil(L);
    lua_pushinteger(L, ret);
    ret = 2;
  }
  return ret;
}

  size_t sz;
  const char* hostname = luaL_checklstring(L, 2, &sz);
  int flags = luaL_optint(L, 3, 0);
  char *peer = NULL;

  int ret = X509_check_host(cert, hostname, sz, flags, &peer);
  ret = openssl_push_check_result(L, ret, peer);
  OPENSSL_free(peer);
  return ret;

  size_t sz;
  const char *email = luaL_checklstring(L, 2, &sz);
  int flags = luaL_optint(L, 3, 0);
  int ret = X509_check_email(cert, email, sz, flags);
  return openssl_push_check_result(L, ret, NULL);
static LUA_FUNCTION(openssl_x509_check_ip)
  size_t sz;
  const char *ip = luaL_checklstring(L, 2, &sz);
  int flags = luaL_optint(L, 3, 0);
  int ret = X509_check_ip(cert, (const unsigned char*)ip, sz, flags);
  return openssl_push_check_result(L, ret, NULL);
  {"check_ip_asc", openssl_x509_check_ip},
#if OPENSSL_VERSION_NUMBER > 0x10002000L
static LuaL_Enumeration check_flags_const[] =
{
#define DEFINE_ENUM(x)  \
  {#x,  X509_CHECK_FLAG_##x}
  DEFINE_ENUM(ALWAYS_CHECK_SUBJECT),
#if OPENSSL_VERSION_NUMBER > 0x10100000L
  DEFINE_ENUM(NEVER_CHECK_SUBJECT),
#endif
  DEFINE_ENUM(NO_WILDCARDS),
  DEFINE_ENUM(NO_PARTIAL_WILDCARDS),
  DEFINE_ENUM(MULTI_LABEL_WILDCARDS),
  DEFINE_ENUM(SINGLE_LABEL_SUBDOMAINS),
#undef DEFINE_ENUM

  {NULL,           0}
};
#endif

#if OPENSSL_VERSION_NUMBER > 0x10002000L
  lua_pushliteral(L, "check_flag");
  lua_newtable(L);
  auxiliar_enumerate(L, -1, check_flags_const);
  lua_settable(L, -3);
#endif