Commit message: Fixed bug: 'luaD_callnoyield' called twice in a row

In luaD_callnoyield, when there is a possible stack overflow, it
zeros the number of CallInfos to force a check when calling the
function. However, if the "function" is not a function, the code will
raise an error before checking the stack. Then, the error handling calls
luaD_callnoyield again and nCcalls is decremented again, crossing the
stack redzone without raising an error. (This loop can only happens
once, because the error handler must be a function.  But once is enough
to cross the redzone.)
CVE: CVE-2020-24342
CWE: CWE-787
Vulnerability description: Lua through 5.4.0 allows a stack redzone cross in luaO_pushvfstring because a protection mechanism wrongly calls luaD_callnoyield twice in a row.

Project: lua
File: ldo.c

void luaD_callnoyield (lua_State *L, StkId func, int nResults) {
  incXCcalls(L);
  if (getCcalls(L) <= CSTACKERR) {  /* possible C stack overflow? */
    luaE_exitCcall(L);  /* to compensate decrement in next call */
    luaE_enterCcall(L);  /* check properly */
  }
  luaD_call(L, func, nResults);
  decXCcalls(L);
}

Project: lua
File: ldo.c

static StkId rethook (lua_State *L, CallInfo *ci, StkId firstres, int nres) {
  ptrdiff_t oldtop = savestack(L, L->top);  /* hook may change top */
  int delta = 0;
  if (isLuacode(ci)) {
    Proto *p = ci_func(ci)->p;
    if (p->is_vararg)
      delta = ci->u.l.nextraargs + p->numparams + 1;
    if (L->top < ci->top)
      L->top = ci->top;  /* correct top to run hook */
  }
  if (L->hookmask & LUA_MASKRET) {  /* is return hook on? */
    int ftransfer;
    ci->func += delta;  /* if vararg, back to virtual 'func' */
    ftransfer = cast(unsigned short, firstres - ci->func);
    luaD_hook(L, LUA_HOOKRET, -1, ftransfer, nres);  /* call it */
    ci->func -= delta;
  }
  if (isLua(ci = ci->previous))
    L->oldpc = pcRel(ci->u.l.savedpc, ci_func(ci)->p);  /* update 'oldpc' */
  return restorestack(L, oldtop);
}

Project: lua
File: ldo.c

LUA_API int lua_resume (lua_State *L, lua_State *from, int nargs,
                                      int *nresults) {
  int status;
  lua_lock(L);
  if (L->status == LUA_OK) {  /* may be starting a coroutine */
    if (L->ci != &L->base_ci)  /* not in base level? */
      return resume_error(L, "cannot resume non-suspended coroutine", nargs);
    else if (L->top - (L->ci->func + 1) == nargs)  /* no function? */
      return resume_error(L, "cannot resume dead coroutine", nargs);
  }
  else if (L->status != LUA_YIELD)  /* ended with errors? */
    return resume_error(L, "cannot resume dead coroutine", nargs);
  if (from == NULL)
    L->nCcalls = CSTACKTHREAD;
  else  /* correct 'nCcalls' for this thread */
    L->nCcalls = getCcalls(from) - L->nci - CSTACKCF;
  if (L->nCcalls <= CSTACKERR)
    return resume_error(L, "C stack overflow", nargs);
  luai_userstateresume(L, nargs);
  api_checknelems(L, (L->status == LUA_OK) ? nargs + 1 : nargs);
  status = luaD_rawrunprotected(L, resume, &nargs);
   /* continue running after recoverable errors */
  while (errorstatus(status) && recover(L, status)) {
    /* unroll continuation */
    status = luaD_rawrunprotected(L, unroll, &status);
  }
  if (likely(!errorstatus(status)))
    lua_assert(status == L->status);  /* normal end or yield */
  else {  /* unrecoverable error */
    L->status = cast_byte(status);  /* mark thread as 'dead' */
    luaD_seterrorobj(L, status, L->top);  /* push error message */
    L->ci->top = L->top;
  }
  *nresults = (status == LUA_YIELD) ? L->ci->u2.nyield
                                    : cast_int(L->top - (L->ci->func + 1));
  lua_unlock(L);
  return status;
}