Commit message: None
CVE: CVE-2016-3819
CWE: CWE-119
Vulnerability description: Integer overflow in codecs/on2/h264dec/source/h264bsd_dpb.c in libstagefright in mediaserver in Android 4.x before 4.4.4, 5.0.x before 5.0.2, 5.1.x before 5.1.1, and 6.x before 2016-08-01 allows remote attackers to execute arbitrary code or cause a denial of service (memory corruption) via a crafted media file, aka internal bug 28533562.

Project: Android

static i32 ComparePictures(const void *ptr1, const void *ptr2)
{

/* Variables */

 dpbPicture_t *pic1, *pic2;

/* Code */

    ASSERT(ptr1);
    ASSERT(ptr2);

    pic1 = (dpbPicture_t*)ptr1;
    pic2 = (dpbPicture_t*)ptr2;

 /* both are non-reference pictures, check if needed for display */
 if (!IS_REFERENCE(*pic1) && !IS_REFERENCE(*pic2))
 {
 if (pic1->toBeDisplayed && !pic2->toBeDisplayed)
 return(-1);
 else if (!pic1->toBeDisplayed && pic2->toBeDisplayed)
 return(1);
 else
 return(0);
 }
 /* only pic 1 needed for reference -> greater */
 else if (!IS_REFERENCE(*pic2))
 return(-1);
 /* only pic 2 needed for reference -> greater */
 else if (!IS_REFERENCE(*pic1))
 return(1);
 /* both are short term reference pictures -> check picNum */
 else if (IS_SHORT_TERM(*pic1) && IS_SHORT_TERM(*pic2))
 {
 if (pic1->picNum > pic2->picNum)
 return(-1);
 else if (pic1->picNum < pic2->picNum)
 return(1);
 else
 return(0);
 }
 /* only pic 1 is short term -> greater */
 else if (IS_SHORT_TERM(*pic1))
 return(-1);
 /* only pic 2 is short term -> greater */
 else if (IS_SHORT_TERM(*pic2))
 return(1);
 /* both are long term reference pictures -> check picNum (contains the
     * longTermPicNum */
 else
 {
 if (pic1->picNum > pic2->picNum)
 return(1);
 else if (pic1->picNum < pic2->picNum)
 return(-1);
 else
 return(0);
 }
}

Project: Android

static i32 FindDpbPic(dpbStorage_t *dpb, i32 picNum, u32 isShortTerm)
{

/* Variables */

    u32 i = 0;
    u32 found = HANTRO_FALSE;

/* Code */

 if (isShortTerm)
 {
 while (i < dpb->maxRefFrames && !found)
 {
 if (IS_SHORT_TERM(dpb->buffer[i]) &&
              dpb->buffer[i].picNum == picNum)
                found = HANTRO_TRUE;
 else
                i++;
 }
 }
 else
 {
        ASSERT(picNum >= 0);
 while (i < dpb->maxRefFrames && !found)
 {
 if (IS_LONG_TERM(dpb->buffer[i]) &&
              dpb->buffer[i].picNum == picNum)
                found = HANTRO_TRUE;
 else
                i++;
 }
 }

 if (found)
 return((i32)i);
 else
 return(-1);

}

Project: Android

dpbPicture_t* FindSmallestPicOrderCnt(dpbStorage_t *dpb)
{

/* Variables */

    u32 i;
    i32 picOrderCnt;
 dpbPicture_t *tmp;

/* Code */

    ASSERT(dpb);

    picOrderCnt = 0x7FFFFFFF;
    tmp = NULL;

 for (i = 0; i <= dpb->dpbSize; i++)
 {
 if (dpb->buffer[i].toBeDisplayed &&
 (dpb->buffer[i].picOrderCnt < picOrderCnt))
 {
            tmp = dpb->buffer + i;
            picOrderCnt = dpb->buffer[i].picOrderCnt;
 }
 }

 return(tmp);

}

Project: Android

static u32 Mmcop1(dpbStorage_t *dpb, u32 currPicNum, u32 differenceOfPicNums)
{

/* Variables */

    i32 index, picNum;

/* Code */

    ASSERT(currPicNum < dpb->maxFrameNum);

    picNum = (i32)currPicNum - (i32)differenceOfPicNums;

    index = FindDpbPic(dpb, picNum, HANTRO_TRUE);
 if (index < 0)
 return(HANTRO_NOK);

    SET_UNUSED(dpb->buffer[index]);
    dpb->numRefFrames--;
 if (!dpb->buffer[index].toBeDisplayed)
        dpb->fullness--;

 return(HANTRO_OK);

}

Project: Android

static u32 Mmcop2(dpbStorage_t *dpb, u32 longTermPicNum)
{

/* Variables */

    i32 index;

/* Code */

    index = FindDpbPic(dpb, (i32)longTermPicNum, HANTRO_FALSE);
 if (index < 0)
 return(HANTRO_NOK);

    SET_UNUSED(dpb->buffer[index]);
    dpb->numRefFrames--;
 if (!dpb->buffer[index].toBeDisplayed)
        dpb->fullness--;

 return(HANTRO_OK);

}

Project: Android

static u32 Mmcop3(dpbStorage_t *dpb, u32 currPicNum, u32 differenceOfPicNums,
    u32 longTermFrameIdx)
{

/* Variables */

    i32 index, picNum;
    u32 i;

/* Code */

    ASSERT(dpb);
    ASSERT(currPicNum < dpb->maxFrameNum);

 if ( (dpb->maxLongTermFrameIdx == NO_LONG_TERM_FRAME_INDICES) ||
 (longTermFrameIdx > dpb->maxLongTermFrameIdx) )
 return(HANTRO_NOK);

 /* check if a long term picture with the same longTermFrameIdx already
     * exist and remove it if necessary */
 for (i = 0; i < dpb->maxRefFrames; i++)
 if (IS_LONG_TERM(dpb->buffer[i]) &&
 (u32)dpb->buffer[i].picNum == longTermFrameIdx)
 {
            SET_UNUSED(dpb->buffer[i]);
            dpb->numRefFrames--;
 if (!dpb->buffer[i].toBeDisplayed)
                dpb->fullness--;
 break;
 }

    picNum = (i32)currPicNum - (i32)differenceOfPicNums;

    index = FindDpbPic(dpb, picNum, HANTRO_TRUE);
 if (index < 0)
 return(HANTRO_NOK);
 if (!IS_EXISTING(dpb->buffer[index]))
 return(HANTRO_NOK);

    dpb->buffer[index].status = LONG_TERM;
    dpb->buffer[index].picNum = (i32)longTermFrameIdx;

 return(HANTRO_OK);

}

Project: Android

static u32 Mmcop4(dpbStorage_t *dpb, u32 maxLongTermFrameIdx)
{

/* Variables */

    u32 i;

/* Code */

    dpb->maxLongTermFrameIdx = maxLongTermFrameIdx;

 for (i = 0; i < dpb->maxRefFrames; i++)
 if (IS_LONG_TERM(dpb->buffer[i]) &&
 ( ((u32)dpb->buffer[i].picNum > maxLongTermFrameIdx) ||
 (dpb->maxLongTermFrameIdx == NO_LONG_TERM_FRAME_INDICES) ) )
 {
            SET_UNUSED(dpb->buffer[i]);
            dpb->numRefFrames--;
 if (!dpb->buffer[i].toBeDisplayed)
                dpb->fullness--;
 }

 return(HANTRO_OK);

}

Project: Android

static u32 Mmcop5(dpbStorage_t *dpb)
{

/* Variables */

    u32 i;

/* Code */

 for (i = 0; i < 16; i++)
 {
 if (IS_REFERENCE(dpb->buffer[i]))
 {
            SET_UNUSED(dpb->buffer[i]);
 if (!dpb->buffer[i].toBeDisplayed)
                dpb->fullness--;
 }
 }

 /* output all pictures */
 while (OutputPicture(dpb) == HANTRO_OK)
 ;
    dpb->numRefFrames = 0;
    dpb->maxLongTermFrameIdx = NO_LONG_TERM_FRAME_INDICES;
    dpb->prevRefFrameNum = 0;

 return(HANTRO_OK);

}

Project: Android

static void SetPicNums(dpbStorage_t *dpb, u32 currFrameNum)
{

/* Variables */

    u32 i;
    i32 frameNumWrap;

/* Code */

    ASSERT(dpb);
    ASSERT(currFrameNum < dpb->maxFrameNum);

 for (i = 0; i < dpb->numRefFrames; i++)
 if (IS_SHORT_TERM(dpb->buffer[i]))
 {
 if (dpb->buffer[i].frameNum > currFrameNum)
                frameNumWrap =
 (i32)dpb->buffer[i].frameNum - (i32)dpb->maxFrameNum;
 else
                frameNumWrap = (i32)dpb->buffer[i].frameNum;
            dpb->buffer[i].picNum = frameNumWrap;
 }

}

Project: Android

static void ShellSort(dpbPicture_t *pPic, u32 num)
{

    u32 i, j;
    u32 step;
 dpbPicture_t tmpPic;

    step = 7;

 while (step)
 {
 for (i = step; i < num; i++)
 {
            tmpPic = pPic[i];
            j = i;
 while (j >= step && ComparePictures(pPic + j - step, &tmpPic) > 0)
 {
                pPic[j] = pPic[j-step];
                j -= step;
 }
            pPic[j] = tmpPic;
 }
        step >>= 1;
 }

}

Project: Android

static u32 SlidingWindowRefPicMarking(dpbStorage_t *dpb)
{

/* Variables */

    i32 index, picNum;
    u32 i;

/* Code */

 if (dpb->numRefFrames < dpb->maxRefFrames)
 {
 return(HANTRO_OK);
 }
 else
 {
        index = -1;
        picNum = 0;
 /* find the oldest short term picture */
 for (i = 0; i < dpb->numRefFrames; i++)
 if (IS_SHORT_TERM(dpb->buffer[i]))
 if (dpb->buffer[i].picNum < picNum || index == -1)
 {
                    index = (i32)i;
                    picNum = dpb->buffer[i].picNum;
 }
 if (index >= 0)
 {
            SET_UNUSED(dpb->buffer[index]);
            dpb->numRefFrames--;
 if (!dpb->buffer[index].toBeDisplayed)
                dpb->fullness--;

 return(HANTRO_OK);
 }
 }

 return(HANTRO_NOK);

}

Project: Android

u8* h264bsdAllocateDpbImage(dpbStorage_t *dpb)
{

/* Variables */

/* Code */

    ASSERT( !dpb->buffer[dpb->dpbSize].toBeDisplayed &&
 !IS_REFERENCE(dpb->buffer[dpb->dpbSize]) );
    ASSERT(dpb->fullness <=  dpb->dpbSize);

    dpb->currentOut = dpb->buffer + dpb->dpbSize;

 return(dpb->currentOut->data);

}

Project: Android

void h264bsdFlushDpb(dpbStorage_t *dpb)
{

 /* don't do anything if buffer not reserved */
 if (dpb->buffer)
 {
        dpb->flushed = 1;
 /* output all pictures */
 while (OutputPicture(dpb) == HANTRO_OK)
 ;
 }

}

Project: Android

u8* h264bsdGetRefPicData(dpbStorage_t *dpb, u32 index)
{

/* Variables */

/* Code */

 if(index > 16 || dpb->list[index] == NULL)
 return(NULL);
 else if(!IS_EXISTING(*dpb->list[index]))
 return(NULL);
 else
 return(dpb->list[index]->data);

}

Project: Android

void h264bsdInitRefPicList(dpbStorage_t *dpb)
{

/* Variables */

    u32 i;

/* Code */

 for (i = 0; i < dpb->numRefFrames; i++)
        dpb->list[i] = &dpb->buffer[i];

}

Project: Android

u32 h264bsdMarkDecRefPic(
 dpbStorage_t *dpb,
 decRefPicMarking_t *mark,
 image_t *image,
  u32 frameNum,
  i32 picOrderCnt,
  u32 isIdr,
  u32 currentPicId,
  u32 numErrMbs)
{

/* Variables */

    u32 i, status;
    u32 markedAsLongTerm;
    u32 toBeDisplayed;

/* Code */

    ASSERT(dpb);
    ASSERT(mark || !isIdr);
    ASSERT(!isIdr || (frameNum == 0 && picOrderCnt == 0));
    ASSERT(frameNum < dpb->maxFrameNum);

 if (image->data != dpb->currentOut->data)
 {
        EPRINT("TRYING TO MARK NON-ALLOCATED IMAGE");
 return(HANTRO_NOK);
 }

    dpb->lastContainsMmco5 = HANTRO_FALSE;
    status = HANTRO_OK;

    toBeDisplayed = dpb->noReordering ? HANTRO_FALSE : HANTRO_TRUE;

 /* non-reference picture, stored for display reordering purposes */
 if (mark == NULL)
 {
        dpb->currentOut->status = UNUSED;
        dpb->currentOut->frameNum = frameNum;
        dpb->currentOut->picNum = (i32)frameNum;
        dpb->currentOut->picOrderCnt = picOrderCnt;
        dpb->currentOut->toBeDisplayed = toBeDisplayed;
 if (!dpb->noReordering)
            dpb->fullness++;
 }
 /* IDR picture */
 else if (isIdr)
 {

 /* h264bsdCheckGapsInFrameNum not called for IDR pictures -> have to
         * reset numOut and outIndex here */
        dpb->numOut = dpb->outIndex = 0;

 /* flush the buffer */
 Mmcop5(dpb);
 /* if noOutputOfPriorPicsFlag was set -> the pictures preceding the
         * IDR picture shall not be output -> set output buffer empty */
 if (mark->noOutputOfPriorPicsFlag || dpb->noReordering)
 {
            dpb->numOut = 0;
            dpb->outIndex = 0;
 }

 if (mark->longTermReferenceFlag)
 {
            dpb->currentOut->status = LONG_TERM;
            dpb->maxLongTermFrameIdx = 0;
 }
 else
 {
            dpb->currentOut->status = SHORT_TERM;
            dpb->maxLongTermFrameIdx = NO_LONG_TERM_FRAME_INDICES;
 }
        dpb->currentOut->frameNum  = 0;
        dpb->currentOut->picNum    = 0;
        dpb->currentOut->picOrderCnt = 0;
        dpb->currentOut->toBeDisplayed = toBeDisplayed;
        dpb->fullness = 1;
        dpb->numRefFrames = 1;
 }
 /* reference picture */
 else
 {
        markedAsLongTerm = HANTRO_FALSE;
 if (mark->adaptiveRefPicMarkingModeFlag)
 {
            i = 0;
 while (mark->operation[i].memoryManagementControlOperation)
 {
 switch (mark->operation[i].memoryManagementControlOperation)
 {
 case 1:
                        status = Mmcop1(
                          dpb,
                          frameNum,
                          mark->operation[i].differenceOfPicNums);
 break;

 case 2:
                        status = Mmcop2(dpb, mark->operation[i].longTermPicNum);
 break;

 case 3:
                        status = Mmcop3(
                          dpb,
                          frameNum,
                          mark->operation[i].differenceOfPicNums,
                          mark->operation[i].longTermFrameIdx);
 break;

 case 4:
                        status = Mmcop4(
                          dpb,
                          mark->operation[i].maxLongTermFrameIdx);
 break;

 case 5:
                        status = Mmcop5(dpb);
                        dpb->lastContainsMmco5 = HANTRO_TRUE;
                        frameNum = 0;
 break;

 case 6:
                        status = Mmcop6(
                          dpb,
                          frameNum,
                          picOrderCnt,
                          mark->operation[i].longTermFrameIdx);
 if (status == HANTRO_OK)
                            markedAsLongTerm = HANTRO_TRUE;
 break;

 default: /* invalid memory management control operation */
                        status = HANTRO_NOK;
 break;
 }
 if (status != HANTRO_OK)
 {
 break;
 }
                i++;
 }
 }
 else
 {
            status = SlidingWindowRefPicMarking(dpb);
 }
 /* if current picture was not marked as long-term reference by
         * memory management control operation 6 -> mark current as short
         * term and insert it into dpb (if there is room) */
 if (!markedAsLongTerm)
 {
 if (dpb->numRefFrames < dpb->maxRefFrames)
 {
                dpb->currentOut->frameNum = frameNum;
                dpb->currentOut->picNum   = (i32)frameNum;
                dpb->currentOut->picOrderCnt = picOrderCnt;
                dpb->currentOut->status   = SHORT_TERM;
                dpb->currentOut->toBeDisplayed = toBeDisplayed;
                dpb->fullness++;
                dpb->numRefFrames++;
 }
 /* no room */
 else
 {
                status = HANTRO_NOK;
 }
 }
 }

    dpb->currentOut->isIdr = isIdr;
    dpb->currentOut->picId = currentPicId;
    dpb->currentOut->numErrMbs = numErrMbs;

 /* dpb was initialized to not to reorder the pictures -> output current
     * picture immediately */
 if (dpb->noReordering)
 {
        ASSERT(dpb->numOut == 0);
        ASSERT(dpb->outIndex == 0);
        dpb->outBuf[dpb->numOut].data  = dpb->currentOut->data;
        dpb->outBuf[dpb->numOut].isIdr = dpb->currentOut->isIdr;
        dpb->outBuf[dpb->numOut].picId = dpb->currentOut->picId;
        dpb->outBuf[dpb->numOut].numErrMbs = dpb->currentOut->numErrMbs;
        dpb->numOut++;
 }
 else
 {
 /* output pictures if buffer full */
 while (dpb->fullness > dpb->dpbSize)
 {
            i = OutputPicture(dpb);
            ASSERT(i == HANTRO_OK);
 }
 }

 /* sort dpb */
 ShellSort(dpb->buffer, dpb->dpbSize+1);

 return(status);

}

Project: Android

u32 h264bsdReorderRefPicList(
 dpbStorage_t *dpb,
 refPicListReordering_t *order,
  u32 currFrameNum,
  u32 numRefIdxActive)
{

/* Variables */

    u32 i, j, k, picNumPred, refIdx;
    i32 picNum, picNumNoWrap, index;
    u32 isShortTerm;

/* Code */

    ASSERT(order);
    ASSERT(currFrameNum <= dpb->maxFrameNum);
    ASSERT(numRefIdxActive <= MAX_NUM_REF_IDX_L0_ACTIVE);

 /* set dpb picture numbers for sorting */
 SetPicNums(dpb, currFrameNum);

 if (!order->refPicListReorderingFlagL0)
 return(HANTRO_OK);

    refIdx     = 0;
    picNumPred = currFrameNum;

    i = 0;
 while (order->command[i].reorderingOfPicNumsIdc < 3)
 {
 /* short term */
 if (order->command[i].reorderingOfPicNumsIdc < 2)
 {
 if (order->command[i].reorderingOfPicNumsIdc == 0)
 {
                picNumNoWrap =
 (i32)picNumPred - (i32)order->command[i].absDiffPicNum;
 if (picNumNoWrap < 0)
                    picNumNoWrap += (i32)dpb->maxFrameNum;
 }
 else
 {
                picNumNoWrap =
 (i32)(picNumPred + order->command[i].absDiffPicNum);
 if (picNumNoWrap >= (i32)dpb->maxFrameNum)
                    picNumNoWrap -= (i32)dpb->maxFrameNum;
 }
            picNumPred = (u32)picNumNoWrap;
            picNum = picNumNoWrap;
 if ((u32)picNumNoWrap > currFrameNum)
                picNum -= (i32)dpb->maxFrameNum;
            isShortTerm = HANTRO_TRUE;
 }
 /* long term */
 else
 {
            picNum = (i32)order->command[i].longTermPicNum;
            isShortTerm = HANTRO_FALSE;

 }
 /* find corresponding picture from dpb */
        index = FindDpbPic(dpb, picNum, isShortTerm);
 if (index < 0 || !IS_EXISTING(dpb->buffer[index]))
 return(HANTRO_NOK);

 /* shift pictures */
 for (j = numRefIdxActive; j > refIdx; j--)
            dpb->list[j] = dpb->list[j-1];
 /* put picture into the list */
        dpb->list[refIdx++] = &dpb->buffer[index];
 /* remove later references to the same picture */
 for (j = k = refIdx; j <= numRefIdxActive; j++)
 if(dpb->list[j] != &dpb->buffer[index])
                dpb->list[k++] = dpb->list[j];

        i++;
 }

 return(HANTRO_OK);

}

Project: Android

u32 h264bsdResetDpb(
 dpbStorage_t *dpb,
  u32 picSizeInMbs,
  u32 dpbSize,
  u32 maxRefFrames,
  u32 maxFrameNum,
  u32 noReordering)
{

/* Code */

    ASSERT(picSizeInMbs);
    ASSERT(maxRefFrames <= MAX_NUM_REF_PICS);
    ASSERT(maxRefFrames <= dpbSize);
    ASSERT(maxFrameNum);
    ASSERT(dpbSize);

    h264bsdFreeDpb(dpb);

 return h264bsdInitDpb(dpb, picSizeInMbs, dpbSize, maxRefFrames,
                          maxFrameNum, noReordering);
}