Commit message: fix some exploitable overflows (#994, #997)
CVE: CVE-2018-7752
CWE: CWE-119
Vulnerability description: GPAC through 0.7.1 has a Buffer Overflow in the gf_media_avc_read_sps function in media_tools/av_parsers.c, a different vulnerability than CVE-2018-1000100.

Project: gpac
File: avc_ext.c

void AVC_RewriteESDescriptor(GF_MPEGVisualSampleEntryBox *avc)
{
	AVC_RewriteESDescriptorEx(avc, NULL);
}

Project: gpac
File: avc_ext.c

GF_Box *btrt_New()
{
	GF_BitRateBox *tmp = (GF_BitRateBox *) gf_malloc(sizeof(GF_BitRateBox));
	if (tmp == NULL) return NULL;
	memset(tmp, 0, sizeof(GF_BitRateBox));
	tmp->type = GF_ISOM_BOX_TYPE_BTRT;
	return (GF_Box *)tmp;
}

Project: gpac
File: avc_ext.c

static GF_Err gf_isom_avc_config_update_ex(GF_ISOFile *the_file, u32 trackNumber, u32 DescriptionIndex, GF_AVCConfig *cfg, u32 op_type)
{
	GF_TrackBox *trak;
	GF_Err e;
	GF_MPEGVisualSampleEntryBox *entry;

	e = CanAccessMovie(the_file, GF_ISOM_OPEN_WRITE);
	if (e) return e;
	trak = gf_isom_get_track_from_file(the_file, trackNumber);
	if (!trak || !trak->Media || !DescriptionIndex) return GF_BAD_PARAM;
	entry = (GF_MPEGVisualSampleEntryBox *)gf_list_get(trak->Media->information->sampleTable->SampleDescription->other_boxes, DescriptionIndex-1);
	if (!entry) return GF_BAD_PARAM;
	switch (entry->type) {
	case GF_ISOM_BOX_TYPE_AVC1:
	case GF_ISOM_BOX_TYPE_AVC2:
	case GF_ISOM_BOX_TYPE_AVC3:
	case GF_ISOM_BOX_TYPE_AVC4:
	case GF_ISOM_BOX_TYPE_SVC1:
	case GF_ISOM_BOX_TYPE_MVC1:
		break;
	default:
		return GF_BAD_PARAM;
	}

	switch (op_type) {
	/*AVCC replacement*/
	case 0:
		if (!cfg) return GF_BAD_PARAM;
		if (!entry->avc_config) entry->avc_config = (GF_AVCConfigurationBox*)gf_isom_box_new(GF_ISOM_BOX_TYPE_AVCC);
		if (entry->avc_config->config) gf_odf_avc_cfg_del(entry->avc_config->config);
		entry->avc_config->config = AVC_DuplicateConfig(cfg);
		entry->type = GF_ISOM_BOX_TYPE_AVC1;
		break;
	/*SVCC replacement*/
	case 1:
		if (!cfg) return GF_BAD_PARAM;
		if (!entry->svc_config) entry->svc_config = (GF_AVCConfigurationBox*)gf_isom_box_new(GF_ISOM_BOX_TYPE_SVCC);
		if (entry->svc_config->config) gf_odf_avc_cfg_del(entry->svc_config->config);
		entry->svc_config->config = AVC_DuplicateConfig(cfg);
		entry->type = GF_ISOM_BOX_TYPE_AVC1;
		break;
	/*SVCC replacement and AVC removal*/
	case 2:
		if (!cfg) return GF_BAD_PARAM;
		if (entry->avc_config) {
			gf_isom_box_del((GF_Box*)entry->avc_config);
			entry->avc_config = NULL;
		}
		if (!entry->svc_config) entry->svc_config = (GF_AVCConfigurationBox*)gf_isom_box_new(GF_ISOM_BOX_TYPE_SVCC);
		if (entry->svc_config->config) gf_odf_avc_cfg_del(entry->svc_config->config);
		entry->svc_config->config = AVC_DuplicateConfig(cfg);
		entry->type = GF_ISOM_BOX_TYPE_SVC1;
		break;
	/*AVCC removal and switch to avc3*/
	case 3:
		if (!entry->avc_config || !entry->avc_config->config)
			return GF_BAD_PARAM;

		if (entry->svc_config) {
			gf_isom_box_del((GF_Box*)entry->svc_config);
			entry->svc_config = NULL;
		}
		if (entry->mvc_config) {
			gf_isom_box_del((GF_Box*)entry->mvc_config);
			entry->mvc_config = NULL;
		}

		while (gf_list_count(entry->avc_config->config->sequenceParameterSets)) {
			GF_AVCConfigSlot *sl = (GF_AVCConfigSlot*)gf_list_get(entry->avc_config->config->sequenceParameterSets, 0);
			gf_list_rem(entry->avc_config->config->sequenceParameterSets, 0);
			if (sl->data) gf_free(sl->data);
			gf_free(sl);
		}

		while (gf_list_count(entry->avc_config->config->pictureParameterSets)) {
			GF_AVCConfigSlot *sl = (GF_AVCConfigSlot*)gf_list_get(entry->avc_config->config->pictureParameterSets, 0);
			gf_list_rem(entry->avc_config->config->pictureParameterSets, 0);
			if (sl->data) gf_free(sl->data);
			gf_free(sl);
		}

		if (entry->type == GF_ISOM_BOX_TYPE_AVC1)
			entry->type = GF_ISOM_BOX_TYPE_AVC3;
		else if (entry->type == GF_ISOM_BOX_TYPE_AVC2)
			entry->type = GF_ISOM_BOX_TYPE_AVC4;
		break;
	/*MVCC replacement*/
	case 4:
		if (!cfg) return GF_BAD_PARAM;
		if (!entry->mvc_config) entry->mvc_config = (GF_AVCConfigurationBox*)gf_isom_box_new(GF_ISOM_BOX_TYPE_MVCC);
		if (entry->mvc_config->config) gf_odf_avc_cfg_del(entry->mvc_config->config);
		entry->mvc_config->config = AVC_DuplicateConfig(cfg);
		entry->type = GF_ISOM_BOX_TYPE_AVC1;
		break;
	/*MVCC replacement and AVC removal*/
	case 5:
		if (!cfg) return GF_BAD_PARAM;
		if (entry->avc_config) {
			gf_isom_box_del((GF_Box*)entry->avc_config);
			entry->avc_config = NULL;
		}
		if (!entry->mvc_config) entry->mvc_config = (GF_AVCConfigurationBox*)gf_isom_box_new(GF_ISOM_BOX_TYPE_MVCC);
		if (entry->mvc_config->config) gf_odf_avc_cfg_del(entry->mvc_config->config);
		entry->mvc_config->config = AVC_DuplicateConfig(cfg);
		entry->type = GF_ISOM_BOX_TYPE_MVC1;
		break;
	}
	AVC_RewriteESDescriptor(entry);
	return GF_OK;
}

Project: gpac

GF_Err gf_isom_hevc_config_new(GF_ISOFile *the_file, u32 trackNumber, GF_HEVCConfig *cfg, char *URLname, char *URNname, u32 *outDescriptionIndex)
{
	GF_TrackBox *trak;
	GF_Err e;
	u32 dataRefIndex;
	GF_MPEGVisualSampleEntryBox *entry;

	e = CanAccessMovie(the_file, GF_ISOM_OPEN_WRITE);
	if (e) return e;

	trak = gf_isom_get_track_from_file(the_file, trackNumber);
	if (!trak || !trak->Media || !cfg) return GF_BAD_PARAM;

	e = Media_FindDataRef(trak->Media->information->dataInformation->dref, URLname, URNname, &dataRefIndex);
	if (e) return e;
	if (!dataRefIndex) {
		e = Media_CreateDataRef(trak->Media->information->dataInformation->dref, URLname, URNname, &dataRefIndex);
		if (e) return e;
	}
	if (!the_file->keep_utc)
		trak->Media->mediaHeader->modificationTime = gf_isom_get_mp4time();

	entry = (GF_MPEGVisualSampleEntryBox *) gf_isom_box_new(GF_ISOM_BOX_TYPE_HVC1);
	if (!entry) return GF_OUT_OF_MEM;
	entry->hevc_config = (GF_HEVCConfigurationBox*)gf_isom_box_new(GF_ISOM_BOX_TYPE_HVCC);
	entry->hevc_config->config = HEVC_DuplicateConfig(cfg);
	entry->dataReferenceIndex = dataRefIndex;
	e = gf_list_add(trak->Media->information->sampleTable->SampleDescription->other_boxes, entry);
	*outDescriptionIndex = gf_list_count(trak->Media->information->sampleTable->SampleDescription->other_boxes);
	HEVC_RewriteESDescriptor(entry);
	return e;
}

Project: gpac
File: avc_ext.c

GF_Err gf_isom_oinf_write_entry(void *entry, GF_BitStream *bs)
{
	GF_OperatingPointsInformation* ptr = (GF_OperatingPointsInformation *)entry;
	u32 i, j, count;
	if (!ptr) return GF_OK;

	gf_bs_write_u16(bs, ptr->scalability_mask);
	gf_bs_write_int(bs, 0xFF, 2);//reserved
	count=gf_list_count(ptr->profile_tier_levels);
	gf_bs_write_int(bs, count, 6);
	for (i = 0; i < count; i++) {
		LHEVC_ProfileTierLevel *ptl = (LHEVC_ProfileTierLevel *)gf_list_get(ptr->profile_tier_levels, i);
		gf_bs_write_int(bs, ptl->general_profile_space, 2);
		gf_bs_write_int(bs, ptl->general_tier_flag, 1);
		gf_bs_write_int(bs, ptl->general_profile_idc, 5);
		gf_bs_write_u32(bs, ptl->general_profile_compatibility_flags);
		gf_bs_write_long_int(bs, ptl->general_constraint_indicator_flags, 48);
		gf_bs_write_u8(bs, ptl->general_level_idc);
	}
	count=gf_list_count(ptr->operating_points);
	gf_bs_write_u16(bs, count);
	for (i = 0; i < count; i++) {
		LHEVC_OperatingPoint *op = (LHEVC_OperatingPoint *)gf_list_get(ptr->operating_points, i);;
		gf_bs_write_u16(bs, op->output_layer_set_idx);
		gf_bs_write_u8(bs, op->max_temporal_id);
		gf_bs_write_u8(bs, op->layer_count);
		for (j = 0; j < op->layer_count; j++) {
			gf_bs_write_u8(bs, op->layers_info[j].ptl_idx);
			gf_bs_write_int(bs, op->layers_info[j].layer_id, 6);
			op->layers_info[j].is_outputlayer ? gf_bs_write_int(bs, 0x1, 1)  : gf_bs_write_int(bs, 0x0, 1);
			op->layers_info[j].is_alternate_outputlayer ? gf_bs_write_int(bs, 0x1, 1)  : gf_bs_write_int(bs, 0x0, 1);
		}
		gf_bs_write_u16(bs, op->minPicWidth);
		gf_bs_write_u16(bs, op->minPicHeight);
		gf_bs_write_u16(bs, op->maxPicWidth);
		gf_bs_write_u16(bs, op->maxPicHeight);
		gf_bs_write_int(bs, op->maxChromaFormat, 2);
		gf_bs_write_int(bs, op->maxBitDepth - 8, 3);
		gf_bs_write_int(bs, 0x1, 1);//resereved
		op->frame_rate_info_flag ? gf_bs_write_int(bs, 0x1, 1)  : gf_bs_write_int(bs, 0x0, 1);
		op->bit_rate_info_flag ? gf_bs_write_int(bs, 0x1, 1)  : gf_bs_write_int(bs, 0x0, 1);
		if (op->frame_rate_info_flag) {
			gf_bs_write_u16(bs, op->avgFrameRate);
			gf_bs_write_int(bs, 0xFF, 6); //reserved
			gf_bs_write_int(bs, op->constantFrameRate, 2);
		}
		if (op->bit_rate_info_flag) {
			gf_bs_write_u32(bs, op->maxBitRate);
			gf_bs_write_u32(bs, op->avgBitRate);
		}
	}
	count=gf_list_count(ptr->dependency_layers);
	gf_bs_write_u8(bs, count);
	for (i = 0; i < count; i++) {
		LHEVC_DependentLayer *dep = (LHEVC_DependentLayer *)gf_list_get(ptr->dependency_layers, i);
		gf_bs_write_u8(bs, dep->dependent_layerID);
		gf_bs_write_u8(bs, dep->num_layers_dependent_on);
		for (j = 0; j < dep->num_layers_dependent_on; j++)
			gf_bs_write_u8(bs, dep->dependent_on_layerID[j]);
		for (j = 0; j < 16; j++) {
			if (ptr->scalability_mask & (1 << j))
				gf_bs_write_u8(bs, dep->dimension_identifier[j]);
		}
	}

	return GF_OK;
}

Project: gpac
File: avc_ext.c

GF_Err hvcc_Read(GF_Box *s, GF_BitStream *bs)
{
	u64 pos;
	GF_HEVCConfigurationBox *ptr = (GF_HEVCConfigurationBox *)s;

	if (ptr->config) gf_odf_hevc_cfg_del(ptr->config);

	pos = gf_bs_get_position(bs);
	ptr->config = gf_odf_hevc_cfg_read_bs(bs, (s->type == GF_ISOM_BOX_TYPE_HVCC) ? GF_FALSE : GF_TRUE);
	pos = gf_bs_get_position(bs) - pos ;
	if (pos < ptr->size)
		ptr->size -= (u32) pos;

	return ptr->config ? GF_OK : GF_ISOM_INVALID_FILE;
}

Project: gpac
File: avc_ext.c

GF_Err m4ds_Size(GF_Box *s)
{
	GF_Err e;
	u32 descSize = 0;
	GF_MPEG4ExtensionDescriptorsBox *ptr = (GF_MPEG4ExtensionDescriptorsBox *)s;
	e = gf_odf_desc_list_size(ptr->descriptors, &descSize);
	ptr->size += descSize;
	return e;
}

Project: gpac
File: av_parsers.c

u32 _ilog_(u32 v)
{
	u32 ret=0;
	while(v) {
		ret++;
		v>>=1;
	}
	return(ret);
}