Commit message: None
CVE: CVE-2016-3742
CWE: CWE-20
Vulnerability description: decoder/ih264d_process_intra_mb.c in mediaserver in Android 6.x before 2016-07-01 mishandles intra mode, which allows remote attackers to execute arbitrary code or cause a denial of service (memory corruption) via a crafted media file, aka internal bug 28165659.

Project: Android

WORD32 ih264d_unpack_coeff4x4_4x4blk(dec_struct_t * ps_dec,
                                   WORD16 *pi2_out_coeff_data,
                                   UWORD8 *pu1_inv_scan)
{
 tu_sblk4x4_coeff_data_t *ps_tu_4x4 = (tu_sblk4x4_coeff_data_t *)ps_dec->pv_proc_tu_coeff_data;
    UWORD16 u2_sig_coeff_map = ps_tu_4x4->u2_sig_coeff_map;
    WORD32 idx = 0;
    WORD16 *pi2_coeff_data = &ps_tu_4x4->ai2_level[0];
    WORD32 dc_only_flag = 0;
    WORD32 num_coeff = 0;

    PROFILE_DISABLE_UNPACK_LUMA()
 while(u2_sig_coeff_map)
 {
        idx = CLZ(u2_sig_coeff_map);

        idx = 31 - idx;
        RESET_BIT(u2_sig_coeff_map,idx);

        idx = pu1_inv_scan[idx];
        pi2_out_coeff_data[idx] = *pi2_coeff_data++;
        num_coeff++;
 }

 if((num_coeff == 1) && (idx == 0))
 {
        dc_only_flag = 1;
 }

 {
        WORD32 offset;
        offset = (UWORD8 *)pi2_coeff_data - (UWORD8 *)ps_tu_4x4;
        offset = ALIGN4(offset);
        ps_dec->pv_proc_tu_coeff_data = (void *)((UWORD8 *)ps_dec->pv_proc_tu_coeff_data + offset);
 }

 return dc_only_flag;
}

Project: Android

void ih264d_unpack_coeff4x4_8x8blk_chroma(dec_struct_t * ps_dec,
 dec_mb_info_t * ps_cur_mb_info,
                                          UWORD16 ui2_chroma_csbp,
                                          WORD16 *pi2_out_coeff_data)
{
    UWORD8 *pu1_inv_scan;
    UWORD8 u1_mb_field_decoding_flag = ps_cur_mb_info->u1_mb_field_decodingflag;
    UWORD8 u1_field_coding_flag = ps_cur_mb_info->ps_curmb->u1_mb_fld;

    PROFILE_DISABLE_UNPACK_CHROMA()
 if(u1_field_coding_flag || u1_mb_field_decoding_flag)
 {
        pu1_inv_scan = (UWORD8 *)gau1_ih264d_inv_scan_fld;
 }
 else
 {
        pu1_inv_scan = (UWORD8 *)gau1_ih264d_inv_scan;
 }

 if(ui2_chroma_csbp & 0x1)
 {
        memset(pi2_out_coeff_data,0,16*sizeof(WORD16));
        ih264d_unpack_coeff4x4_4x4blk(ps_dec,
                                      pi2_out_coeff_data,
                                      pu1_inv_scan);
 }
    pi2_out_coeff_data += 16;
 if(ui2_chroma_csbp & 0x2)
 {
        memset(pi2_out_coeff_data,0,16*sizeof(WORD16));
        ih264d_unpack_coeff4x4_4x4blk(ps_dec,
                                      pi2_out_coeff_data,
                                      pu1_inv_scan);
 }

    pi2_out_coeff_data += 16;
 if(ui2_chroma_csbp & 0x4)
 {
        memset(pi2_out_coeff_data,0,16*sizeof(WORD16));
        ih264d_unpack_coeff4x4_4x4blk(ps_dec,
                                      pi2_out_coeff_data,
                                      pu1_inv_scan);
 }

    pi2_out_coeff_data += 16;
 if(ui2_chroma_csbp & 0x8)
 {
        memset(pi2_out_coeff_data,0,16*sizeof(WORD16));
        ih264d_unpack_coeff4x4_4x4blk(ps_dec,
                                      pi2_out_coeff_data,
                                      pu1_inv_scan);
 }
}

Project: Android

WORD32 ih264d_unpack_coeff8x8_8x8blk_cavlc(dec_struct_t * ps_dec,
 dec_mb_info_t * ps_cur_mb_info,
                                            UWORD16 ui2_luma_csbp,
                                            WORD16 *pi2_out_coeff_data)
{
    UWORD8 *pu1_inv_scan;
    UWORD8 u1_mb_field_decoding_flag = ps_cur_mb_info->u1_mb_field_decodingflag;
    UWORD8 u1_field_coding_flag = ps_cur_mb_info->ps_curmb->u1_mb_fld;
    WORD32 dc_only_flag = 0;

    PROFILE_DISABLE_UNPACK_LUMA()
 if(ui2_luma_csbp & 0x33)
 {
        memset(pi2_out_coeff_data,0,64*sizeof(WORD16));
 }

 if(!u1_mb_field_decoding_flag)
 {
        pu1_inv_scan =
 (UWORD8*)gau1_ih264d_inv_scan_prog8x8_cavlc[0];
 }
 else
 {
        pu1_inv_scan =
 (UWORD8*)gau1_ih264d_inv_scan_int8x8_cavlc[0];
 }
 if(ui2_luma_csbp & 0x1)
 {
        dc_only_flag = ih264d_unpack_coeff4x4_4x4blk(ps_dec,
                                      pi2_out_coeff_data,
                                      pu1_inv_scan);
 }

 if(!u1_mb_field_decoding_flag)
 {
        pu1_inv_scan =
 (UWORD8*)gau1_ih264d_inv_scan_prog8x8_cavlc[1];
 }
 else
 {
        pu1_inv_scan =
 (UWORD8*)gau1_ih264d_inv_scan_int8x8_cavlc[1];
 }
 if(ui2_luma_csbp & 0x2)
 {
        dc_only_flag = 0;
        ih264d_unpack_coeff4x4_4x4blk(ps_dec,
                                      pi2_out_coeff_data,
                                      pu1_inv_scan);
 }

 if(!u1_mb_field_decoding_flag)
 {
        pu1_inv_scan =
 (UWORD8*)gau1_ih264d_inv_scan_prog8x8_cavlc[2];
 }
 else
 {
        pu1_inv_scan =
 (UWORD8*)gau1_ih264d_inv_scan_int8x8_cavlc[2];
 }
 if(ui2_luma_csbp & 0x10)
 {
        dc_only_flag = 0;
        ih264d_unpack_coeff4x4_4x4blk(ps_dec,
                                      pi2_out_coeff_data,
                                      pu1_inv_scan);
 }

 if(!u1_mb_field_decoding_flag)
 {
        pu1_inv_scan =
 (UWORD8*)gau1_ih264d_inv_scan_prog8x8_cavlc[3];
 }
 else
 {
        pu1_inv_scan =
 (UWORD8*)gau1_ih264d_inv_scan_int8x8_cavlc[3];
 }
 if(ui2_luma_csbp & 0x20)
 {
        dc_only_flag = 0;
        ih264d_unpack_coeff4x4_4x4blk(ps_dec,
                                      pi2_out_coeff_data,
                                      pu1_inv_scan);
 }
 return dc_only_flag;
}

Project: Android

UWORD32 ih264d_unpack_luma_coeff4x4_mb(dec_struct_t * ps_dec,
 dec_mb_info_t * ps_cur_mb_info,
                                    UWORD8 intra_flag)
{
    UWORD8 u1_mb_type = ps_cur_mb_info->u1_mb_type;
    UWORD16 ui2_luma_csbp = ps_cur_mb_info->u2_luma_csbp;
    UWORD8 *pu1_inv_scan = ps_dec->pu1_inv_scan;
    WORD16 *pi2_coeff_data = ps_dec->pi2_coeff_data;

    PROFILE_DISABLE_UNPACK_LUMA()
 if(!ps_cur_mb_info->u1_tran_form8x8)
 {
        UWORD32 u4_luma_dc_only_csbp = 0;
        UWORD32 u4_temp = 0;
        WORD16* pi2_dc_val = NULL;
 /*
         * Reserve the pointer to dc vals. The dc vals will be copied
         * after unpacking of ac vals since memset to 0 inside.
         */
 if(intra_flag && (u1_mb_type != I_4x4_MB))
 {
 if(CHECKBIT(ps_cur_mb_info->u1_yuv_dc_block_flag,0))
 {
                pi2_dc_val = (WORD16 *)ps_dec->pv_proc_tu_coeff_data;

                ps_dec->pv_proc_tu_coeff_data = (void *)(pi2_dc_val + 16);
 }
 }

 if(ui2_luma_csbp)
 {
            pi2_coeff_data = ps_dec->pi2_coeff_data;
            u4_temp = ih264d_unpack_coeff4x4_8x8blk(ps_dec,
                                          ps_cur_mb_info,
                                          ui2_luma_csbp,
                                          pi2_coeff_data);
            u4_luma_dc_only_csbp = u4_temp;

            pi2_coeff_data += 32;

            ui2_luma_csbp = ui2_luma_csbp >> 2;
            u4_temp = ih264d_unpack_coeff4x4_8x8blk(ps_dec,
                                          ps_cur_mb_info,
                                          ui2_luma_csbp,
                                          pi2_coeff_data);

            u4_luma_dc_only_csbp |= (u4_temp << 2);

            pi2_coeff_data += 32 + 64;

            ui2_luma_csbp = ui2_luma_csbp >> 6;
            u4_temp = ih264d_unpack_coeff4x4_8x8blk(ps_dec,
                                          ps_cur_mb_info,
                                          ui2_luma_csbp,
                                          pi2_coeff_data);

            u4_luma_dc_only_csbp |= (u4_temp << 8);

            pi2_coeff_data += 32;

            ui2_luma_csbp = ui2_luma_csbp >> 2;
            u4_temp = ih264d_unpack_coeff4x4_8x8blk(ps_dec,
                                          ps_cur_mb_info,
                                          ui2_luma_csbp,
                                          pi2_coeff_data);
            u4_luma_dc_only_csbp |= (u4_temp << 10);
 }

 if(pi2_dc_val != NULL)
 {
            WORD32 i;
            pi2_coeff_data = ps_dec->pi2_coeff_data;
 for(i = 0; i < 4; i++)
 {
                pi2_coeff_data[0] = pi2_dc_val[0];
                pi2_coeff_data[4 * 16] = pi2_dc_val[4];
                pi2_coeff_data[8 * 16] = pi2_dc_val[8];
                pi2_coeff_data[12 * 16] = pi2_dc_val[12];

                pi2_dc_val++; /* Point to next column */
                pi2_coeff_data += 16;
 }
            u4_luma_dc_only_csbp = ps_cur_mb_info->u2_luma_csbp ^ 0xFFFF;
 }
 return u4_luma_dc_only_csbp;
 }
 else
 {
        UWORD32 u4_luma_dc_only_cbp = 0;
        WORD32 dc_only_flag;
 if(ui2_luma_csbp)
 {
            pi2_coeff_data = ps_dec->pi2_coeff_data;
            dc_only_flag = ih264d_unpack_coeff8x8_8x8blk_cavlc(ps_dec,
                                          ps_cur_mb_info,
                                          ui2_luma_csbp,
                                          pi2_coeff_data);
            INSERT_BIT(u4_luma_dc_only_cbp, 0, dc_only_flag);

            pi2_coeff_data += 64;

            ui2_luma_csbp = ui2_luma_csbp >> 2;
            dc_only_flag = ih264d_unpack_coeff8x8_8x8blk_cavlc(ps_dec,
                                          ps_cur_mb_info,
                                          ui2_luma_csbp,
                                          pi2_coeff_data);

            INSERT_BIT(u4_luma_dc_only_cbp, 1, dc_only_flag);

            pi2_coeff_data += 64;

            ui2_luma_csbp = ui2_luma_csbp >> 6;
            dc_only_flag = ih264d_unpack_coeff8x8_8x8blk_cavlc(ps_dec,
                                          ps_cur_mb_info,
                                          ui2_luma_csbp,
                                          pi2_coeff_data);

            INSERT_BIT(u4_luma_dc_only_cbp, 2, dc_only_flag);

            pi2_coeff_data += 64;
            ui2_luma_csbp = ui2_luma_csbp >> 2;
            dc_only_flag = ih264d_unpack_coeff8x8_8x8blk_cavlc(ps_dec,
                                          ps_cur_mb_info,
                                          ui2_luma_csbp,
                                          pi2_coeff_data);
            INSERT_BIT(u4_luma_dc_only_cbp, 3, dc_only_flag);
 }
 return u4_luma_dc_only_cbp;
 }

}

Project: Android

UWORD32 ih264d_unpack_luma_coeff8x8_mb(dec_struct_t * ps_dec,
 dec_mb_info_t * ps_cur_mb_info)
{
    WORD32 blk_8x8_cnt;
    WORD16 *pi2_out_coeff_data = ps_dec->pi2_coeff_data;
    UWORD8 u1_field_coding_flag = ps_cur_mb_info->ps_curmb->u1_mb_fld;
    UWORD8 *pu1_inv_scan;
    UWORD32 u4_luma_dc_only_cbp = 0;

    PROFILE_DISABLE_UNPACK_LUMA()
 if(!u1_field_coding_flag)
 {
 /*******************************************************************/
 /* initializing inverse scan  matrices                             */
 /*******************************************************************/
        pu1_inv_scan = (UWORD8 *)gau1_ih264d_inv_scan_prog8x8_cabac;
 }
 else
 {
 /*******************************************************************/
 /* initializing inverse scan  matrices                             */
 /*******************************************************************/
        pu1_inv_scan = (UWORD8 *)gau1_ih264d_inv_scan_int8x8_cabac;
 }

 for(blk_8x8_cnt = 0; blk_8x8_cnt < 4; blk_8x8_cnt++)
 {
 if(CHECKBIT(ps_cur_mb_info->u1_cbp, blk_8x8_cnt))
 {
 tu_blk8x8_coeff_data_t *ps_tu_8x8 = (tu_blk8x8_coeff_data_t *)ps_dec->pv_proc_tu_coeff_data;
            UWORD32 u4_sig_coeff_map;
            WORD32 idx = 0;
            WORD16 *pi2_coeff_data = &ps_tu_8x8->ai2_level[0];
            WORD32 num_coeff = 0;

 /* memset 64 coefficient to zero */
            memset(pi2_out_coeff_data,0,64*sizeof(WORD16));

            u4_sig_coeff_map = ps_tu_8x8->au4_sig_coeff_map[1];

 while(u4_sig_coeff_map)
 {
                idx = CLZ(u4_sig_coeff_map);

                idx = 31 - idx;
                RESET_BIT(u4_sig_coeff_map,idx);

                idx = pu1_inv_scan[idx + 32];
                pi2_out_coeff_data[idx] = *pi2_coeff_data++;
                num_coeff++;
 }

            u4_sig_coeff_map = ps_tu_8x8->au4_sig_coeff_map[0];
 while(u4_sig_coeff_map)
 {
                idx = CLZ(u4_sig_coeff_map);

                idx = 31 - idx;
                RESET_BIT(u4_sig_coeff_map,idx);

                idx = pu1_inv_scan[idx];
                pi2_out_coeff_data[idx] = *pi2_coeff_data++;
                num_coeff++;
 }

 if((num_coeff == 1) && (idx == 0))
 {
                SET_BIT(u4_luma_dc_only_cbp,blk_8x8_cnt);
 }


 {
                WORD32 offset;
                offset = (UWORD8 *)pi2_coeff_data - (UWORD8 *)ps_tu_8x8;
                offset = ALIGN4(offset);
                ps_dec->pv_proc_tu_coeff_data = (void *)((UWORD8 *)ps_dec->pv_proc_tu_coeff_data + offset);
 }
 }
        pi2_out_coeff_data += 64;
 }

 return u4_luma_dc_only_cbp;
}