Commit message: Fix write heap buffer overflow in opj_mqc_byteout(). Discovered by Ke Liu of Tencent's Xuanwu LAB (#835)
CVE: CVE-2016-10504
CWE: CWE-119
Vulnerability description: Heap-based buffer overflow vulnerability in the opj_mqc_byteout function in mqc.c in OpenJPEG before 2.2.0 allows remote attackers to cause a denial of service (application crash) via a crafted bmp file.

Project: openjpeg
File: tcd.c

static OPJ_BOOL opj_tcd_code_block_dec_allocate(opj_tcd_cblk_dec_t *
        p_code_block)
{
    if (! p_code_block->data) {

        p_code_block->data = (OPJ_BYTE*) opj_malloc(OPJ_COMMON_DEFAULT_CBLK_DATA_SIZE);
        if (! p_code_block->data) {
            return OPJ_FALSE;
        }
        p_code_block->data_max_size = OPJ_COMMON_DEFAULT_CBLK_DATA_SIZE;
        /*fprintf(stderr, "Allocate 8192 elements of code_block->data\n");*/

        p_code_block->segs = (opj_tcd_seg_t *) opj_calloc(OPJ_J2K_DEFAULT_NB_SEGS,
                             sizeof(opj_tcd_seg_t));
        if (! p_code_block->segs) {
            return OPJ_FALSE;
        }
        /*fprintf(stderr, "Allocate %d elements of code_block->data\n", OPJ_J2K_DEFAULT_NB_SEGS * sizeof(opj_tcd_seg_t));*/

        p_code_block->m_current_max_segs = OPJ_J2K_DEFAULT_NB_SEGS;
        /*fprintf(stderr, "m_current_max_segs of code_block->data = %d\n", p_code_block->m_current_max_segs);*/
    } else {
        /* sanitize */
        OPJ_BYTE* l_data = p_code_block->data;
        OPJ_UINT32 l_data_max_size = p_code_block->data_max_size;
        opj_tcd_seg_t * l_segs = p_code_block->segs;
        OPJ_UINT32 l_current_max_segs = p_code_block->m_current_max_segs;

        memset(p_code_block, 0, sizeof(opj_tcd_cblk_dec_t));
        p_code_block->data = l_data;
        p_code_block->data_max_size = l_data_max_size;
        p_code_block->segs = l_segs;
        p_code_block->m_current_max_segs = l_current_max_segs;
    }

    return OPJ_TRUE;
}

Project: openjpeg
File: tcd.c

static void opj_tcd_code_block_dec_deallocate(opj_tcd_precinct_t * p_precinct)
{
    OPJ_UINT32 cblkno, l_nb_code_blocks;

    opj_tcd_cblk_dec_t * l_code_block = p_precinct->cblks.dec;
    if (l_code_block) {
        /*fprintf(stderr,"deallocate codeblock:{\n");*/
        /*fprintf(stderr,"\t x0=%d, y0=%d, x1=%d, y1=%d\n",l_code_block->x0, l_code_block->y0, l_code_block->x1, l_code_block->y1);*/
        /*fprintf(stderr,"\t numbps=%d, numlenbits=%d, len=%d, numnewpasses=%d, real_num_segs=%d, m_current_max_segs=%d\n ",
                        l_code_block->numbps, l_code_block->numlenbits, l_code_block->len, l_code_block->numnewpasses, l_code_block->real_num_segs, l_code_block->m_current_max_segs );*/


        l_nb_code_blocks = p_precinct->block_size / sizeof(opj_tcd_cblk_dec_t);
        /*fprintf(stderr,"nb_code_blocks =%d\t}\n", l_nb_code_blocks);*/

        for (cblkno = 0; cblkno < l_nb_code_blocks; ++cblkno) {

            if (l_code_block->data) {
                opj_free(l_code_block->data);
                l_code_block->data = 00;
            }

            if (l_code_block->segs) {
                opj_free(l_code_block->segs);
                l_code_block->segs = 00;
            }

            ++l_code_block;
        }

        opj_free(p_precinct->cblks.dec);
        p_precinct->cblks.dec = 00;
    }
}

Project: openjpeg
File: tcd.c

static OPJ_BOOL opj_tcd_dc_level_shift_decode(opj_tcd_t *p_tcd)
{
    OPJ_UINT32 compno;
    opj_tcd_tilecomp_t * l_tile_comp = 00;
    opj_tccp_t * l_tccp = 00;
    opj_image_comp_t * l_img_comp = 00;
    opj_tcd_resolution_t* l_res = 00;
    opj_tcd_tile_t * l_tile;
    OPJ_UINT32 l_width, l_height, i, j;
    OPJ_INT32 * l_current_ptr;
    OPJ_INT32 l_min, l_max;
    OPJ_UINT32 l_stride;

    l_tile = p_tcd->tcd_image->tiles;
    l_tile_comp = l_tile->comps;
    l_tccp = p_tcd->tcp->tccps;
    l_img_comp = p_tcd->image->comps;

    for (compno = 0; compno < l_tile->numcomps; compno++) {
        l_res = l_tile_comp->resolutions + l_img_comp->resno_decoded;
        l_width = (OPJ_UINT32)(l_res->x1 - l_res->x0);
        l_height = (OPJ_UINT32)(l_res->y1 - l_res->y0);
        l_stride = (OPJ_UINT32)(l_tile_comp->x1 - l_tile_comp->x0) - l_width;

        assert(l_height == 0 ||
               l_width + l_stride <= l_tile_comp->data_size / l_height); /*MUPDF*/

        if (l_img_comp->sgnd) {
            l_min = -(1 << (l_img_comp->prec - 1));
            l_max = (1 << (l_img_comp->prec - 1)) - 1;
        } else {
            l_min = 0;
            l_max = (1 << l_img_comp->prec) - 1;
        }

        l_current_ptr = l_tile_comp->data;

        if (l_tccp->qmfbid == 1) {
            for (j = 0; j < l_height; ++j) {
                for (i = 0; i < l_width; ++i) {
                    *l_current_ptr = opj_int_clamp(*l_current_ptr + l_tccp->m_dc_level_shift, l_min,
                                                   l_max);
                    ++l_current_ptr;
                }
                l_current_ptr += l_stride;
            }
        } else {
            for (j = 0; j < l_height; ++j) {
                for (i = 0; i < l_width; ++i) {
                    OPJ_FLOAT32 l_value = *((OPJ_FLOAT32 *) l_current_ptr);
                    OPJ_INT32 l_value_int = (OPJ_INT32)opj_lrintf(l_value);
                    if (l_value > INT_MAX ||
                            (l_value_int > 0 && l_tccp->m_dc_level_shift > 0 &&
                             l_value_int > INT_MAX - l_tccp->m_dc_level_shift)) {
                        *l_current_ptr = l_max;
                    } else {
                        *l_current_ptr = opj_int_clamp(
                                             l_value_int + l_tccp->m_dc_level_shift, l_min, l_max);
                    }
                    ++l_current_ptr;
                }
                l_current_ptr += l_stride;
            }
        }

        ++l_img_comp;
        ++l_tccp;
        ++l_tile_comp;
    }

    return OPJ_TRUE;
}