Commit message: fix recursion level crash (bb #3706).

Thanks to Stephane Chazelas for the analysis.
CVE: CVE-2011-3627
CWE: CWE-189
Vulnerability description: The bytecode engine in ClamAV before 0.97.3 allows remote attackers to cause a denial of service (crash) via vectors related to "recursion level" and (1) libclamav/bytecode.c and (2) libclamav/bytecode_api.c.

Project: clamav-devel
File: bytecode_api.c

int32_t cli_bcapi_extract_new(struct cli_bc_ctx *ctx, int32_t id)
{
    cli_ctx *cctx;
    int res = -1;

    cli_event_count(EV, BCEV_EXTRACTED);
    cli_dbgmsg("previous tempfile had %u bytes\n", ctx->written);
    if (!ctx->written)
	return 0;
    if (ctx->ctx && cli_updatelimits(ctx->ctx, ctx->written))
	return -1;
    ctx->written = 0;
    lseek(ctx->outfd, 0, SEEK_SET);
    cli_dbgmsg("bytecode: scanning extracted file %s\n", ctx->tempfile);
    cctx = (cli_ctx*)ctx->ctx;
    if (cctx) {
	cli_file_t current = cctx->container_type;
	if (ctx->containertype != CL_TYPE_ANY)
	    cctx->container_type = ctx->containertype;
	res = cli_magic_scandesc(ctx->outfd, cctx);
	cctx->container_type = current;
	if (res == CL_VIRUS) {
	    if (cctx->virname)
		ctx->virname = *cctx->virname;
	    ctx->found = 1;
	}
    }
    if ((cctx && cctx->engine->keeptmp) ||
	(ftruncate(ctx->outfd, 0) == -1)) {

	close(ctx->outfd);
	if (!(cctx && cctx->engine->keeptmp) && ctx->tempfile)
	    cli_unlink(ctx->tempfile);
	free(ctx->tempfile);
	ctx->tempfile = NULL;
	ctx->outfd = 0;
    }
    cli_dbgmsg("bytecode: extracting new file with id %u\n", id);
    return res;
}

Project: clamav-devel
File: bytecode.c

int cli_bytecode_runhook(cli_ctx *cctx, const struct cl_engine *engine, struct cli_bc_ctx *ctx,
			 unsigned id, fmap_t *map, const char **virname)
{
    const unsigned *hooks = engine->hooks[id - _BC_START_HOOKS];
    unsigned i, hooks_cnt = engine->hooks_cnt[id - _BC_START_HOOKS];
    int ret;
    unsigned executed = 0, breakflag = 0, errorflag = 0;

    cli_dbgmsg("Bytecode executing hook id %u (%u hooks)\n", id, hooks_cnt);
    /* restore match counts */
    cli_bytecode_context_setfile(ctx, map);
    ctx->hooks.match_counts = ctx->lsigcnt;
    ctx->hooks.match_offsets = ctx->lsigoff;
    for (i=0;i < hooks_cnt;i++) {
	const struct cli_bc *bc = &engine->bcs.all_bcs[hooks[i]];
	if (bc->lsig) {
	    if (!cctx->hook_lsig_matches ||
		!cli_bitset_test(cctx->hook_lsig_matches, bc->hook_lsig_id-1))
		continue;
	    cli_dbgmsg("Bytecode: executing bytecode %u (lsig matched)\n" , bc->id);
	}
	cli_bytecode_context_setfuncid(ctx, bc, 0);
	ret = cli_bytecode_run(&engine->bcs, bc, ctx);
	executed++;
	if (ret != CL_SUCCESS) {
	    cli_warnmsg("Bytecode %u failed to run: %s\n", bc->id, cl_strerror(ret));
	    errorflag = 1;
	    continue;
	}
	if (ctx->virname) {
	    cli_dbgmsg("Bytecode found virus: %s\n", ctx->virname);
	    if (virname)
		*virname = ctx->virname;
	    cli_bytecode_context_clear(ctx);
	    return CL_VIRUS;
	}
	ret = cli_bytecode_context_getresult_int(ctx);
	/* TODO: use prefix here */
	cli_dbgmsg("Bytecode %u returned %u\n", bc->id, ret);
	if (ret == 0xcea5e) {
	    cli_dbgmsg("Bytecode set BREAK flag in hook!\n");
	    breakflag = 1;
	}
	if (!ret) {
	    char *tempfile;
	    int fd = cli_bytecode_context_getresult_file(ctx, &tempfile);
	    if (fd && fd != -1) {
		if (cctx && cctx->engine->keeptmp)
		    cli_dbgmsg("Bytecode %u unpacked file saved in %s\n",
			       bc->id, tempfile);
		else
		    cli_dbgmsg("Bytecode %u unpacked file\n", bc->id);
		lseek(fd, 0, SEEK_SET);
		cli_dbgmsg("***** Scanning unpacked file ******\n");
		ret = cli_magic_scandesc(fd, cctx);
		if (!cctx || !cctx->engine->keeptmp)
		    if (ftruncate(fd, 0) == -1)
			cli_dbgmsg("ftruncate failed on %d\n", fd);
		close(fd);
		if (!cctx || !cctx->engine->keeptmp) {
		    if (tempfile && cli_unlink(tempfile))
			ret = CL_EUNLINK;
		}
		free(tempfile);
		if (ret != CL_CLEAN) {
		    if (ret == CL_VIRUS)
			cli_dbgmsg("Scanning unpacked file by bytecode %u found a virus\n", bc->id);
		    cli_bytecode_context_clear(ctx);
		    return ret;
		}
		cli_bytecode_context_reset(ctx);
		continue;
	    }
	}
	cli_bytecode_context_reset(ctx);
    }
    if (executed)
	cli_dbgmsg("Bytecode: executed %u bytecodes for this hook\n", executed);
    else
	cli_dbgmsg("Bytecode: no logical signature matched, no bytecode executed\n");
    if (errorflag && cctx && cctx->engine->bytecode_mode == CL_BYTECODE_MODE_TEST)
	return CL_EBYTECODE_TESTFAIL;
    return breakflag ? CL_BREAK : CL_CLEAN;
}