Commit message: fixed #1887
CVE: CVE-2021-40566
CWE: CWE-416
Vulnerability description: A Segmentation fault casued by heap use after free vulnerability exists in Gpac through 1.0.1 via the mpgviddmx_process function in reframe_mpgvid.c when using mp4box, which causes a denial of service.

Project: gpac
File: dec_xvid.c

const GF_FilterRegister *xviddec_register(GF_FilterSession *session)
{
#if !defined(GPAC_DISABLE_AV_PARSERS) && defined(GPAC_HAS_XVID)
	return &XVIDRegister;
#else
	return NULL;
#endif
}

Project: gpac
File: dec_xvid.c

static void xviddec_finalize(GF_Filter *filter)
{
	GF_XVIDCtx *ctx = gf_filter_get_udta(filter);
	if (ctx->codec) xvid_decore(ctx->codec, XVID_DEC_DESTROY, NULL, NULL);
	while (gf_list_count(ctx->src_packets)) {
		GF_FilterPacket *pck = gf_list_pop_back(ctx->src_packets);
		gf_filter_pck_unref(pck);
	}
	gf_list_del(ctx->src_packets);
}

Project: gpac
File: dec_xvid.c

static GF_Err xviddec_initialize(GF_Filter *filter)
{
	GF_XVIDCtx *ctx = gf_filter_get_udta(filter);
	if (!xvid_is_init) {
#ifdef XVID_USE_OLD_API
		XVID_INIT_PARAM init;
		init.api_version = 0;
		init.core_build = 0;
		/*get info*/
		init.cpu_flags = XVID_CPU_CHKONLY;
		xvid_init(NULL, 0, &init, NULL);
		/*then init*/
		xvid_init(NULL, 0, &init, NULL);
#else
		xvid_gbl_init_t init;
		init.debug = 0;
		init.version = XVID_VERSION;
		init.cpu_flags = 0; /*autodetect*/
		xvid_global(NULL, 0, &init, NULL);
#endif
		xvid_is_init = GF_TRUE;
	}

#ifndef XVID_USE_OLD_API
	if (ctx->film_effect) ctx->base_filters |= XVID_FILMEFFECT;
#endif

#ifdef XVID_USE_OLD_API
	if (ctx->deblock_y) ctx->base_filters |= XVID_DEC_DEBLOCKY;
#else
	if (ctx->deblock_y) ctx->base_filters |= XVID_DEBLOCKY;
#endif

#ifdef XVID_USE_OLD_API
	if (ctx->deblock_uv) ctx->base_filters |= XVID_DEC_DEBLOCKUV;
#else
	if (ctx->deblock_uv) ctx->base_filters |= XVID_DEBLOCKUV;
#endif

#ifndef XVID_USE_OLD_API
	if (ctx->dering_y) ctx->base_filters |= XVID_DERINGY | XVID_DEBLOCKY;
	if (ctx->dering_uv) ctx->base_filters |= XVID_DERINGUV | XVID_DEBLOCKUV;
#endif
	ctx->src_packets = gf_list_new();
	return GF_OK;
}

Project: gpac
File: dec_xvid.c

static GF_Err xviddec_configure_pid(GF_Filter *filter, GF_FilterPid *pid, Bool is_remove)
{
	const GF_PropertyValue *p;
	GF_M4VDecSpecInfo dsi;
	GF_Err e;
	Bool is_first = GF_FALSE;
#ifdef XVID_USE_OLD_API
	XVID_DEC_FRAME frame;
	XVID_DEC_PARAM par;
#else
	xvid_dec_frame_t frame;
	xvid_dec_create_t par;
#endif
	GF_XVIDCtx *ctx = gf_filter_get_udta(filter);

	if (is_remove) {
		if (ctx->opid) {
			gf_filter_pid_remove(ctx->opid);
			ctx->opid = NULL;
		}
		ctx->ipid = NULL;
		return GF_OK;
	}
	if (! gf_filter_pid_check_caps(pid))
		return GF_NOT_SUPPORTED;

	ctx->ipid = pid;
	if (!ctx->opid) {
		ctx->opid = gf_filter_pid_new(filter);
		gf_filter_pid_set_framing_mode(ctx->ipid, GF_TRUE);
		is_first = GF_TRUE;
	}
	//copy properties at init or reconfig
	gf_filter_pid_copy_properties(ctx->opid, ctx->ipid);
	gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_CODECID, &PROP_UINT(GF_CODECID_RAW) );

	p = gf_filter_pid_get_property(pid, GF_PROP_PID_DECODER_CONFIG);
	if (p && p->value.data.ptr && p->value.data.size) {
		u32 ex_crc = gf_crc_32(p->value.data.ptr, p->value.data.size);
		if (ctx->cfg_crc == ex_crc) return GF_OK;

		//shoud we flush ?
		if (ctx->codec) xvid_decore(ctx->codec, XVID_DEC_DESTROY, NULL, NULL);
		ctx->codec = NULL;

		ctx->cfg_crc = ex_crc;
	} else if (!is_first) {
		GF_LOG(GF_LOG_ERROR, GF_LOG_CODEC, ("[XVID] Reconfiguring without DSI not yet supported\n"));
		return GF_NOT_SUPPORTED;
	} else {
		return GF_OK;
	}

	/*decode DSI*/
	e = gf_m4v_get_config(p->value.data.ptr, p->value.data.size, &dsi);
	if (e) return e;
	if (!dsi.width || (dsi.width%2) || !dsi.height) return GF_NON_COMPLIANT_BITSTREAM;

	memset(&par, 0, sizeof(par));
	par.width = dsi.width;
	par.height = dsi.height;
	/*note that this may be irrelevant when used through systems (FPS is driven by systems CTS)*/
	ctx->FPS = dsi.clock_rate;
	ctx->FPS /= 1000;
	if (!ctx->FPS) ctx->FPS = 30.0f;
	ctx->pixel_ar.num = dsi.par_num;
	ctx->pixel_ar.den = dsi.par_den;

#ifndef XVID_USE_OLD_API
	par.version = XVID_VERSION;
#endif

	if (xvid_decore(NULL, XVID_DEC_CREATE, &par, NULL) < 0) return GF_NON_COMPLIANT_BITSTREAM;

	ctx->width = par.width;
	ctx->height = par.height;
	ctx->codec = par.handle;

	/*init decoder*/
	memset(&frame, 0, sizeof(frame));
	frame.bitstream = (void *) p->value.data.ptr;
	frame.length = p->value.data.size;
#ifndef XVID_USE_OLD_API
	frame.version = XVID_VERSION;
	xvid_decore(ctx->codec, XVID_DEC_DECODE, &frame, NULL);
#else
	/*don't perform error check, XviD doesn't like DSI only frame ...*/
	xvid_decore(ctx->codec, XVID_DEC_DECODE, &frame, NULL);
#endif

	ctx->first_frame = GF_TRUE;
	ctx->out_size = ctx->width * ctx->height * 3 / 2;

	gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_WIDTH, &PROP_UINT(ctx->width) );
	gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_HEIGHT, &PROP_UINT(ctx->height) );
	gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_STRIDE, &PROP_UINT(ctx->width) );
	gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_PAR, &PROP_FRAC(ctx->pixel_ar) );
	gf_filter_pid_set_property(ctx->opid, GF_PROP_PID_PIXFMT, &PROP_UINT(GF_PIXEL_YUV) );

	return GF_OK;
}

Project: gpac
File: dec_xvid.c

static GF_Err xviddec_process(GF_Filter *filter)
{
#ifdef XVID_USE_OLD_API
	XVID_DEC_FRAME frame;
#else
	xvid_dec_frame_t frame;
#endif
	u8 *buffer;
	u32 i, count;
	Bool is_seek;
#if 0
	s32 postproc;
#endif
	s32 res;
	GF_XVIDCtx *ctx = gf_filter_get_udta(filter);
	GF_FilterPacket *pck, *pck_ref, *src_pck, *dst_pck;

	pck = gf_filter_pid_get_packet(ctx->ipid);

	if (!ctx->codec)
		return ctx->cfg_crc ? GF_SERVICE_ERROR : GF_OK;

	memset(&frame, 0, sizeof(frame));
	if (pck) {
		u64 cts = gf_filter_pck_get_cts(pck);;
		frame.bitstream = (char *) gf_filter_pck_get_data(pck, &frame.length);

		//append in cts order since we get output frames in cts order
		pck_ref = pck;
		gf_filter_pck_ref_props(&pck_ref);
		count = gf_list_count(ctx->src_packets);
		src_pck = NULL;
		for (i=0; i<count; i++) {
			u64 acts;
			src_pck = gf_list_get(ctx->src_packets, i);
			acts = gf_filter_pck_get_cts(src_pck);
			if (acts==cts) {
				gf_filter_pck_unref(pck_ref);
				break;
			}
			if (acts>cts) {
				gf_list_insert(ctx->src_packets, pck_ref, i);
				break;
			}
			src_pck = NULL;
		}
		if (!src_pck)
			gf_list_add(ctx->src_packets, pck_ref);


	} else {
		frame.bitstream = NULL;
		frame.length = -1;
	}

packed_frame :

	dst_pck = gf_filter_pck_new_alloc(ctx->opid, ctx->width*ctx->height*3/2, &buffer);
	if (!dst_pck) return GF_OUT_OF_MEM;

#ifdef XVID_USE_OLD_API
	frame.colorspace = XVID_CSP_I420;
	frame.stride = ctx->width;
	frame.image = (void *) buffer;
#else
	frame.version = XVID_VERSION;
	frame.output.csp = XVID_CSP_I420;
	frame.output.stride[0] = ctx->width;
	frame.output.plane[0] = (void *) buffer;
#endif


#if 0
	postproc = ctx->base_filters;
	/*to check, not convinced yet by results...*/
	switch (mmlevel) {
	case GF_CODEC_LEVEL_SEEK:
	case GF_CODEC_LEVEL_DROP:
		/*turn off all post-proc*/
#ifdef XVID_USE_OLD_API
		postproc &= ~XVID_DEC_DEBLOCKY;
		postproc &= ~XVID_DEC_DEBLOCKUV;
#else
		postproc &= ~XVID_DEBLOCKY;
		postproc &= ~XVID_DEBLOCKUV;
		postproc &= ~XVID_FILMEFFECT;
#endif
		break;
	case GF_CODEC_LEVEL_VERY_LATE:
		/*turn off post-proc*/
#ifdef XVID_USE_OLD_API
		postproc &= ~XVID_DEC_DEBLOCKY;
#else
		postproc &= ~XVID_FILMEFFECT;
		postproc &= ~XVID_DEBLOCKY;
#endif
		break;
	case GF_CODEC_LEVEL_LATE:
#ifdef XVID_USE_OLD_API
		postproc &= ~XVID_DEC_DEBLOCKUV;
#else
		postproc &= ~XVID_DEBLOCKUV;
		postproc &= ~XVID_FILMEFFECT;
#endif
		break;
	}
#endif

	/*xvid may keep the first I frame and force a 1-frame delay, so we simply trick it*/
	if (ctx->first_frame) {
		buffer[0] = 'v';
		buffer[1] = 'o';
		buffer[2] = 'i';
		buffer[3] = 'd';
	}
	src_pck = gf_list_get(ctx->src_packets, 0);

	res = xvid_decore(ctx->codec, XVID_DEC_DECODE, &frame, NULL);
	if (res < 0) {
		gf_filter_pck_discard(dst_pck);
		if (pck) gf_filter_pid_drop_packet(ctx->ipid);
		if (src_pck) {
			gf_filter_pck_unref(src_pck);
			gf_list_pop_front(ctx->src_packets);
		}
		if (gf_filter_pid_is_eos(ctx->ipid)) {
			gf_filter_pid_set_eos(ctx->opid);
			return GF_EOS;
		}
		return pck ? GF_NON_COMPLIANT_BITSTREAM : GF_OK;
	}

	if (ctx->first_frame) {
		ctx->first_frame = GF_FALSE;
		if ((buffer[0] == 'v') && (buffer[1] == 'o') && (buffer[2] == 'i') && (buffer[3] =='d')) {
			gf_filter_pck_discard(dst_pck);
			if (pck) gf_filter_pid_drop_packet(ctx->ipid);
			return GF_OK;
		}
	}

	if (src_pck) {
		gf_filter_pck_merge_properties(src_pck, dst_pck);
		is_seek = gf_filter_pck_get_seek_flag(src_pck);
		ctx->next_cts = gf_filter_pck_get_cts(src_pck);
		gf_filter_pck_set_dts(dst_pck, ctx->next_cts);
		ctx->next_cts += gf_filter_pck_get_duration(src_pck);
		gf_filter_pck_unref(src_pck);
		gf_list_pop_front(ctx->src_packets);
	} else {
		is_seek = 0;
		gf_filter_pck_set_cts(dst_pck, ctx->next_cts);
	}

	if (!pck || !is_seek )
		gf_filter_pck_send(dst_pck);
	else
		gf_filter_pck_discard(dst_pck);

	if (res + 6 < frame.length) {
		frame.bitstream = ((char *)frame.bitstream) + res;
		frame.length -= res;
		goto packed_frame;
	}

	if (pck) {
		gf_filter_pid_drop_packet(ctx->ipid);
	}
	//flush all frames if eos is detected
	else if (gf_filter_pid_is_eos(ctx->ipid)) {
		return xviddec_process(filter);
	}

	return GF_OK;
}