Commit message: None
CVE: CVE-2016-2507
CWE: CWE-119
Vulnerability description: Integer overflow in codecs/on2/h264dec/source/h264bsd_storage.c in libstagefright in mediaserver in Android 4.x before 4.4.4, 5.0.x before 5.0.2, 5.1.x before 5.1.1, and 6.x before 2016-07-01 allows remote attackers to execute arbitrary code or cause a denial of service (memory corruption) via a crafted media file, aka internal bug 28532266.

Project: Android

u32 CheckPps(picParamSet_t *pps, seqParamSet_t *sps)
{

    u32 i;
    u32 picSize;

    picSize = sps->picWidthInMbs * sps->picHeightInMbs;

 /* check slice group params */
 if (pps->numSliceGroups > 1)
 {
 if (pps->sliceGroupMapType == 0)
 {
            ASSERT(pps->runLength);
 for (i = 0; i < pps->numSliceGroups; i++)
 {
 if (pps->runLength[i] > picSize)
 return(HANTRO_NOK);
 }
 }
 else if (pps->sliceGroupMapType == 2)
 {
            ASSERT(pps->topLeft);
            ASSERT(pps->bottomRight);
 for (i = 0; i < pps->numSliceGroups-1; i++)
 {
 if (pps->topLeft[i] > pps->bottomRight[i] ||
                    pps->bottomRight[i] >= picSize)
 return(HANTRO_NOK);

 if ( (pps->topLeft[i] % sps->picWidthInMbs) >
 (pps->bottomRight[i] % sps->picWidthInMbs) )
 return(HANTRO_NOK);
 }
 }
 else if (pps->sliceGroupMapType > 2 && pps->sliceGroupMapType < 6)
 {
 if (pps->sliceGroupChangeRate > picSize)
 return(HANTRO_NOK);
 }
 else if (pps->sliceGroupMapType == 6 &&
                 pps->picSizeInMapUnits < picSize)
 return(HANTRO_NOK);
 }

 return(HANTRO_OK);
}

Project: Android

u32 h264bsdCheckAccessUnitBoundary(
 strmData_t *strm,
 nalUnit_t *nuNext,
 storage_t *storage,
  u32 *accessUnitBoundaryFlag)
{

/* Variables */

    u32 tmp, ppsId, frameNum, idrPicId, picOrderCntLsb;
    i32 deltaPicOrderCntBottom, deltaPicOrderCnt[2];
 seqParamSet_t *sps;
 picParamSet_t *pps;

/* Code */

    ASSERT(strm);
    ASSERT(nuNext);
    ASSERT(storage);
    ASSERT(storage->sps);
    ASSERT(storage->pps);

 /* initialize default output to FALSE */
 *accessUnitBoundaryFlag = HANTRO_FALSE;

 if ( ( (nuNext->nalUnitType > 5) && (nuNext->nalUnitType < 12) ) ||
 ( (nuNext->nalUnitType > 12) && (nuNext->nalUnitType <= 18) ) )
 {
 *accessUnitBoundaryFlag = HANTRO_TRUE;
 return(HANTRO_OK);
 }
 else if ( nuNext->nalUnitType != NAL_CODED_SLICE &&
              nuNext->nalUnitType != NAL_CODED_SLICE_IDR )
 {
 return(HANTRO_OK);
 }

 /* check if this is the very first call to this function */
 if (storage->aub->firstCallFlag)
 {
 *accessUnitBoundaryFlag = HANTRO_TRUE;
        storage->aub->firstCallFlag = HANTRO_FALSE;
 }

 /* get picture parameter set id */
    tmp = h264bsdCheckPpsId(strm, &ppsId);
 if (tmp != HANTRO_OK)
 return(tmp);

 /* store sps and pps in separate pointers just to make names shorter */
    pps = storage->pps[ppsId];
 if ( pps == NULL || storage->sps[pps->seqParameterSetId] == NULL  ||
 (storage->activeSpsId != MAX_NUM_SEQ_PARAM_SETS &&
          pps->seqParameterSetId != storage->activeSpsId &&
          nuNext->nalUnitType != NAL_CODED_SLICE_IDR) )
 return(PARAM_SET_ERROR);
    sps = storage->sps[pps->seqParameterSetId];

 if (storage->aub->nuPrev->nalRefIdc != nuNext->nalRefIdc &&
 (storage->aub->nuPrev->nalRefIdc == 0 || nuNext->nalRefIdc == 0))
 *accessUnitBoundaryFlag = HANTRO_TRUE;

 if ((storage->aub->nuPrev->nalUnitType == NAL_CODED_SLICE_IDR &&
          nuNext->nalUnitType != NAL_CODED_SLICE_IDR) ||
 (storage->aub->nuPrev->nalUnitType != NAL_CODED_SLICE_IDR &&
       nuNext->nalUnitType == NAL_CODED_SLICE_IDR))
 *accessUnitBoundaryFlag = HANTRO_TRUE;

    tmp = h264bsdCheckFrameNum(strm, sps->maxFrameNum, &frameNum);
 if (tmp != HANTRO_OK)
 return(HANTRO_NOK);

 if (storage->aub->prevFrameNum != frameNum)
 {
        storage->aub->prevFrameNum = frameNum;
 *accessUnitBoundaryFlag = HANTRO_TRUE;
 }

 if (nuNext->nalUnitType == NAL_CODED_SLICE_IDR)
 {
        tmp = h264bsdCheckIdrPicId(strm, sps->maxFrameNum, nuNext->nalUnitType,
 &idrPicId);
 if (tmp != HANTRO_OK)
 return(HANTRO_NOK);

 if (storage->aub->nuPrev->nalUnitType == NAL_CODED_SLICE_IDR &&
          storage->aub->prevIdrPicId != idrPicId)
 *accessUnitBoundaryFlag = HANTRO_TRUE;

        storage->aub->prevIdrPicId = idrPicId;
 }

 if (sps->picOrderCntType == 0)
 {
        tmp = h264bsdCheckPicOrderCntLsb(strm, sps, nuNext->nalUnitType,
 &picOrderCntLsb);
 if (tmp != HANTRO_OK)
 return(HANTRO_NOK);

 if (storage->aub->prevPicOrderCntLsb != picOrderCntLsb)
 {
            storage->aub->prevPicOrderCntLsb = picOrderCntLsb;
 *accessUnitBoundaryFlag = HANTRO_TRUE;
 }

 if (pps->picOrderPresentFlag)
 {
            tmp = h264bsdCheckDeltaPicOrderCntBottom(strm, sps,
                nuNext->nalUnitType, &deltaPicOrderCntBottom);
 if (tmp != HANTRO_OK)
 return(tmp);

 if (storage->aub->prevDeltaPicOrderCntBottom !=
                deltaPicOrderCntBottom)
 {
                storage->aub->prevDeltaPicOrderCntBottom =
                    deltaPicOrderCntBottom;
 *accessUnitBoundaryFlag = HANTRO_TRUE;
 }
 }
 }
 else if (sps->picOrderCntType == 1 && !sps->deltaPicOrderAlwaysZeroFlag)
 {
        tmp = h264bsdCheckDeltaPicOrderCnt(strm, sps, nuNext->nalUnitType,
          pps->picOrderPresentFlag, deltaPicOrderCnt);
 if (tmp != HANTRO_OK)
 return(tmp);

 if (storage->aub->prevDeltaPicOrderCnt[0] != deltaPicOrderCnt[0])
 {
            storage->aub->prevDeltaPicOrderCnt[0] = deltaPicOrderCnt[0];
 *accessUnitBoundaryFlag = HANTRO_TRUE;
 }

 if (pps->picOrderPresentFlag)
 if (storage->aub->prevDeltaPicOrderCnt[1] != deltaPicOrderCnt[1])
 {
                storage->aub->prevDeltaPicOrderCnt[1] = deltaPicOrderCnt[1];
 *accessUnitBoundaryFlag = HANTRO_TRUE;
 }
 }

 *storage->aub->nuPrev = *nuNext;

 return(HANTRO_OK);

}

Project: Android

void h264bsdInitStorage(storage_t *pStorage)
{

/* Variables */

/* Code */

    ASSERT(pStorage);

    H264SwDecMemset(pStorage, 0, sizeof(storage_t));

    pStorage->activeSpsId = MAX_NUM_SEQ_PARAM_SETS;
    pStorage->activePpsId = MAX_NUM_PIC_PARAM_SETS;

    pStorage->aub->firstCallFlag = HANTRO_TRUE;
}

Project: Android

u32 h264bsdIsEndOfPicture(storage_t *pStorage)
{

/* Variables */

    u32 i, tmp;

/* Code */

 /* primary picture */
 if (!pStorage->sliceHeader[0].redundantPicCnt)
 {
 if (pStorage->slice->numDecodedMbs == pStorage->picSizeInMbs)
 return(HANTRO_TRUE);
 }
 else
 {
 for (i = 0, tmp = 0; i < pStorage->picSizeInMbs; i++)
            tmp += pStorage->mb[i].decoded ? 1 : 0;

 if (tmp == pStorage->picSizeInMbs)
 return(HANTRO_TRUE);
 }

 return(HANTRO_FALSE);

}

Project: Android

u32 h264bsdIsStartOfPicture(storage_t *pStorage)
{

/* Variables */


/* Code */

 if (pStorage->validSliceInAccessUnit == HANTRO_FALSE)
 return(HANTRO_TRUE);
 else
 return(HANTRO_FALSE);

}

Project: Android

void h264bsdResetStorage(storage_t *pStorage)
{

/* Variables */

    u32 i;

/* Code */

    ASSERT(pStorage);

    pStorage->slice->numDecodedMbs = 0;
    pStorage->slice->sliceId = 0;

 for (i = 0; i < pStorage->picSizeInMbs; i++)
 {
        pStorage->mb[i].sliceId = 0;
        pStorage->mb[i].decoded = 0;
 }

}

Project: Android

u32 h264bsdStorePicParamSet(storage_t *pStorage, picParamSet_t *pPicParamSet)
{

/* Variables */

    u32 id;

/* Code */

    ASSERT(pStorage);
    ASSERT(pPicParamSet);
    ASSERT(pPicParamSet->picParameterSetId < MAX_NUM_PIC_PARAM_SETS);
    ASSERT(pPicParamSet->seqParameterSetId < MAX_NUM_SEQ_PARAM_SETS);

    id = pPicParamSet->picParameterSetId;

 /* pic parameter set with id not used before -> allocate memory */
 if (pStorage->pps[id] == NULL)
 {
        ALLOCATE(pStorage->pps[id], 1, picParamSet_t);
 if (pStorage->pps[id] == NULL)
 return(MEMORY_ALLOCATION_ERROR);
 }
 /* picture parameter set with id equal to id of active pps */
 else if (id == pStorage->activePpsId)
 {
 /* check whether seq param set changes, force re-activation of
         * param set if it does. Set activeSpsId to invalid value to
         * accomplish this */
 if (pPicParamSet->seqParameterSetId != pStorage->activeSpsId)
 {
            pStorage->activePpsId = MAX_NUM_PIC_PARAM_SETS + 1;
 }
 /* free memories allocated for old param set */
        FREE(pStorage->pps[id]->runLength);
        FREE(pStorage->pps[id]->topLeft);
        FREE(pStorage->pps[id]->bottomRight);
        FREE(pStorage->pps[id]->sliceGroupId);
 }
 /* overwrite pic param set other than active one -> free memories
     * allocated for old param set */
 else
 {
        FREE(pStorage->pps[id]->runLength);
        FREE(pStorage->pps[id]->topLeft);
        FREE(pStorage->pps[id]->bottomRight);
        FREE(pStorage->pps[id]->sliceGroupId);
 }

 *pStorage->pps[id] = *pPicParamSet;

 return(HANTRO_OK);

}

Project: Android

u32 h264bsdStoreSeqParamSet(storage_t *pStorage, seqParamSet_t *pSeqParamSet)
{

/* Variables */

    u32 id;

/* Code */

    ASSERT(pStorage);
    ASSERT(pSeqParamSet);
    ASSERT(pSeqParamSet->seqParameterSetId < MAX_NUM_SEQ_PARAM_SETS);

    id = pSeqParamSet->seqParameterSetId;

 /* seq parameter set with id not used before -> allocate memory */
 if (pStorage->sps[id] == NULL)
 {
        ALLOCATE(pStorage->sps[id], 1, seqParamSet_t);
 if (pStorage->sps[id] == NULL)
 return(MEMORY_ALLOCATION_ERROR);
 }
 /* sequence parameter set with id equal to id of active sps */
 else if (id == pStorage->activeSpsId)
 {
 /* if seq parameter set contents changes
         *    -> overwrite and re-activate when next IDR picture decoded
         *    ids of active param sets set to invalid values to force
         *    re-activation. Memories allocated for old sps freed
         * otherwise free memeries allocated for just decoded sps and
         * continue */
 if (h264bsdCompareSeqParamSets(pSeqParamSet, pStorage->activeSps) != 0)
 {
            FREE(pStorage->sps[id]->offsetForRefFrame);
            FREE(pStorage->sps[id]->vuiParameters);
            pStorage->activeSpsId = MAX_NUM_SEQ_PARAM_SETS + 1;
            pStorage->activePpsId = MAX_NUM_PIC_PARAM_SETS + 1;
            pStorage->activeSps = NULL;
            pStorage->activePps = NULL;
 }
 else
 {
            FREE(pSeqParamSet->offsetForRefFrame);
            FREE(pSeqParamSet->vuiParameters);
 return(HANTRO_OK);
 }
 }
 /* overwrite seq param set other than active one -> free memories
     * allocated for old param set */
 else
 {
        FREE(pStorage->sps[id]->offsetForRefFrame);
        FREE(pStorage->sps[id]->vuiParameters);
 }

 *pStorage->sps[id] = *pSeqParamSet;

 return(HANTRO_OK);

}

Project: Android

u32 h264bsdValidParamSets(storage_t *pStorage)
{

/* Variables */

    u32 i;

/* Code */

    ASSERT(pStorage);

 for (i = 0; i < MAX_NUM_PIC_PARAM_SETS; i++)
 {
 if ( pStorage->pps[i] &&
             pStorage->sps[pStorage->pps[i]->seqParameterSetId] &&
 CheckPps(pStorage->pps[i],
                      pStorage->sps[pStorage->pps[i]->seqParameterSetId]) ==
                 HANTRO_OK)
 {
 return(HANTRO_OK);
 }
 }

 return(HANTRO_NOK);

}