CVE-2020-9434: Improper Certificate Validation (CWE-295)
value: openssl_x509_check_ip_asc in lua-openssl 0.7.7-1 mishandles X.509 certificate validation because it uses lua_pushboolean for certain non-boolean return values.
CWE description: The product does not validate, or incorrectly validates, a certificate.
CVSS v3 base score: 9.1
Published: 2020-02-27T23:15Z
Repository: https://github.com/zhaozg/lua-openssl
Language: C
Files changed: src/x509.c
Commit date: 2020-02-27 21:53:32 +0800
Commit message: fix security: GSL-2020-026

Diff: diff --git a/src/x509.c b/src/x509.c
index 267419f..cc5ca52 100644
--- a/src/x509.c
+++ b/src/x509.c
@@ -749,71 +749,101 @@ static LUA_FUNCTION(openssl_x509_check)
 
     return 2;
   }
 }
 
+/***
+The functions return 1 for a successful match, 0 for a failed match and -1 for
+an internal error: typically a memory allocation failure or an ASN.1 decoding
+error.
+
+All functions can also return -2 if the input is malformed. For example,
+X509_check_host() returns -2 if the provided name contains embedded NULs.
+*/
+static int openssl_push_check_result(lua_State *L, int ret, const char* name)
+{
+  switch (ret)
+  {
+  case 1:
+    lua_pushboolean(L, 1);
+    if (name)
+    {
+      lua_pushstring(L, name);
+      ret = 2;
+    }
+    break;
+  case 0:
+    lua_pushboolean(L, 0);
+    ret = 1;
+    break;
+  case -1:
+    lua_pushnil(L);
+    lua_pushliteral(L, "internal");
+    ret = 2;
+  case -2:
+    lua_pushnil(L);
+    lua_pushliteral(L, "malformed");
+    ret = 2;
+  default:
+    lua_pushnil(L);
+    lua_pushinteger(L, ret);
+    ret = 2;
+  }
+  return ret;
+}
+
 #if OPENSSL_VERSION_NUMBER > 0x10002000L
 /***
 check x509 for host (only for openssl 1.0.2 or greater)
 @function check_host
 @tparam string host hostname to check for match match with x509 subject
 @treturn boolean result true if host is present and matches the certificate
 */
 static LUA_FUNCTION(openssl_x509_check_host)
 {
   X509 * cert = CHECK_OBJECT(1, X509, "openssl.x509");
-  if (lua_isstring(L, 2))
-  {
-    const char *hostname = lua_tostring(L, 2);
-    lua_pushboolean(L, X509_check_host(cert, hostname, strlen(hostname), 0, NULL));
-  }
-  else
-  {
-    lua_pushboolean(L, 0);
-  }
-  return 1;
+  size_t sz;
+  const char* hostname = luaL_checklstring(L, 2, &sz);
+  int flags = luaL_optint(L, 3, 0);
+  char *peer = NULL;
+
+  int ret = X509_check_host(cert, hostname, sz, flags, &peer);
+  ret = openssl_push_check_result(L, ret, peer);
+  OPENSSL_free(peer);
+  return ret;
 }
+
 /***
 check x509 for email address (only for openssl 1.0.2 or greater)
 @tparam string email to check for match match with x509 subject
 @treturn boolean result true if host is present and matches the certificate
 @function check_email
 */
 static LUA_FUNCTION(openssl_x509_check_email)
 {
   X509 * cert = CHECK_OBJECT(1, X509, "openssl.x509");
-  if (lua_isstring(L, 2))
-  {
-    const char *email = lua_tostring(L, 2);
-    lua_pushboolean(L, X509_check_email(cert, email, strlen(email), 0));
-  }
-  else
-  {
-    lua_pushboolean(L, 0);
-  }
-  return 1;
+  size_t sz;
+  const char *email = luaL_checklstring(L, 2, &sz);
+  int flags = luaL_optint(L, 3, 0);
+  int ret = X509_check_email(cert, email, sz, flags);
+  return openssl_push_check_result(L, ret, NULL);
 }
 
 /***
 check x509 for ip address (ipv4 or ipv6, only for openssl 1.0.2 or greater)
 @function check_ip_asc
 @tparam string ip to check for match match with x509 subject
 @treturn boolean result true if host is present and matches the certificate
 */
-static LUA_FUNCTION(openssl_x509_check_ip_asc)
+static LUA_FUNCTION(openssl_x509_check_ip)
 {
   X509 * cert = CHECK_OBJECT(1, X509, "openssl.x509");
-  if (lua_isstring(L, 2))
-  {
-    const char *ip_asc = lua_tostring(L, 2);
-    lua_pushboolean(L, X509_check_ip_asc(cert, ip_asc, 0));
-  }
-  else
-  {
-    lua_pushboolean(L, 0);
-  }
-  return 1;
+  size_t sz;
+  const char *ip = luaL_checklstring(L, 2, &sz);
+  int flags = luaL_optint(L, 3, 0);
+  int ret = X509_check_ip(cert, (const unsigned char*)ip, sz, flags);
+  return openssl_push_check_result(L, ret, NULL);
 }
 #endif
 
 IMP_LUA_SK(X509, x509)
 
@@ -1358,11 +1388,11 @@ static luaL_Reg x509_funcs[] =
   {"export",      openssl_x509_export},
   {"check",       openssl_x509_check},
 #if OPENSSL_VERSION_NUMBER > 0x10002000L
   {"check_host",  openssl_x509_check_host},
   {"check_email", openssl_x509_check_email},
-  {"check_ip_asc", openssl_x509_check_ip_asc},
+  {"check_ip_asc", openssl_x509_check_ip},
 #endif
   {"pubkey",      openssl_x509_public_key},
   {"version",     openssl_x509_version},
 
   {"__gc",        openssl_x509_free},
@@ -1382,10 +1412,29 @@ static luaL_Reg x509_funcs[] =
   {"verify",     openssl_x509_verify},
 
   {NULL,      NULL},
 };
 
+#if OPENSSL_VERSION_NUMBER > 0x10002000L
+static LuaL_Enumeration check_flags_const[] =
+{
+#define DEFINE_ENUM(x)  \
+  {#x,  X509_CHECK_FLAG_##x}
+  DEFINE_ENUM(ALWAYS_CHECK_SUBJECT),
+#if OPENSSL_VERSION_NUMBER > 0x10100000L
+  DEFINE_ENUM(NEVER_CHECK_SUBJECT),
+#endif
+  DEFINE_ENUM(NO_WILDCARDS),
+  DEFINE_ENUM(NO_PARTIAL_WILDCARDS),
+  DEFINE_ENUM(MULTI_LABEL_WILDCARDS),
+  DEFINE_ENUM(SINGLE_LABEL_SUBDOMAINS),
+#undef DEFINE_ENUM
+
+  {NULL,           0}
+};
+#endif
+
 int luaopen_x509(lua_State *L)
 {
   auxiliar_newclass(L, "openssl.x509", x509_funcs);
 
   lua_newtable(L);
@@ -1409,7 +1458,14 @@ int luaopen_x509(lua_State *L)
 
   lua_pushliteral(L, "version");    /** version */
   lua_pushliteral(L, MYVERSION);
   lua_settable(L, -3);
 
+#if OPENSSL_VERSION_NUMBER > 0x10002000L
+  lua_pushliteral(L, "check_flag");
+  lua_newtable(L);
+  auxiliar_enumerate(L, -1, check_flags_const);
+  lua_settable(L, -3);
+#endif
+
   return 1;
 }

Vulnerable code:   if (lua_isstring(L, 2))
  {
    const char *hostname = lua_tostring(L, 2);
    lua_pushboolean(L, X509_check_host(cert, hostname, strlen(hostname), 0, NULL));
  }
  else
  {
    lua_pushboolean(L, 0);
  }
  return 1;
  if (lua_isstring(L, 2))
  {
    const char *email = lua_tostring(L, 2);
    lua_pushboolean(L, X509_check_email(cert, email, strlen(email), 0));
  }
  else
  {
    lua_pushboolean(L, 0);
  }
  return 1;
static LUA_FUNCTION(openssl_x509_check_ip_asc)
  if (lua_isstring(L, 2))
  {
    const char *ip_asc = lua_tostring(L, 2);
    lua_pushboolean(L, X509_check_ip_asc(cert, ip_asc, 0));
  }
  else
  {
    lua_pushboolean(L, 0);
  }
  return 1;
  {"check_ip_asc", openssl_x509_check_ip_asc},
Fixed code: /***
The functions return 1 for a successful match, 0 for a failed match and -1 for
an internal error: typically a memory allocation failure or an ASN.1 decoding
error.

All functions can also return -2 if the input is malformed. For example,
X509_check_host() returns -2 if the provided name contains embedded NULs.
*/
static int openssl_push_check_result(lua_State *L, int ret, const char* name)
{
  switch (ret)
  {
  case 1:
    lua_pushboolean(L, 1);
    if (name)
    {
      lua_pushstring(L, name);
      ret = 2;
    }
    break;
  case 0:
    lua_pushboolean(L, 0);
    ret = 1;
    break;
  case -1:
    lua_pushnil(L);
    lua_pushliteral(L, "internal");
    ret = 2;
  case -2:
    lua_pushnil(L);
    lua_pushliteral(L, "malformed");
    ret = 2;
  default:
    lua_pushnil(L);
    lua_pushinteger(L, ret);
    ret = 2;
  }
  return ret;
}

  size_t sz;
  const char* hostname = luaL_checklstring(L, 2, &sz);
  int flags = luaL_optint(L, 3, 0);
  char *peer = NULL;

  int ret = X509_check_host(cert, hostname, sz, flags, &peer);
  ret = openssl_push_check_result(L, ret, peer);
  OPENSSL_free(peer);
  return ret;

  size_t sz;
  const char *email = luaL_checklstring(L, 2, &sz);
  int flags = luaL_optint(L, 3, 0);
  int ret = X509_check_email(cert, email, sz, flags);
  return openssl_push_check_result(L, ret, NULL);
static LUA_FUNCTION(openssl_x509_check_ip)
  size_t sz;
  const char *ip = luaL_checklstring(L, 2, &sz);
  int flags = luaL_optint(L, 3, 0);
  int ret = X509_check_ip(cert, (const unsigned char*)ip, sz, flags);
  return openssl_push_check_result(L, ret, NULL);
  {"check_ip_asc", openssl_x509_check_ip},
#if OPENSSL_VERSION_NUMBER > 0x10002000L
static LuaL_Enumeration check_flags_const[] =
{
#define DEFINE_ENUM(x)  \
  {#x,  X509_CHECK_FLAG_##x}
  DEFINE_ENUM(ALWAYS_CHECK_SUBJECT),
#if OPENSSL_VERSION_NUMBER > 0x10100000L
  DEFINE_ENUM(NEVER_CHECK_SUBJECT),
#endif
  DEFINE_ENUM(NO_WILDCARDS),
  DEFINE_ENUM(NO_PARTIAL_WILDCARDS),
  DEFINE_ENUM(MULTI_LABEL_WILDCARDS),
  DEFINE_ENUM(SINGLE_LABEL_SUBDOMAINS),
#undef DEFINE_ENUM

  {NULL,           0}
};
#endif

#if OPENSSL_VERSION_NUMBER > 0x10002000L
  lua_pushliteral(L, "check_flag");
  lua_newtable(L);
  auxiliar_enumerate(L, -1, check_flags_const);
  lua_settable(L, -3);
#endif