Commit message: bmp_read_info_header(): reject bmp files with biBitCount == 0 (#983)
CVE: CVE-2017-12982
CWE: CWE-119
Vulnerability description: The bmp_read_info_header function in bin/jp2/convertbmp.c in OpenJPEG 2.2.0 does not reject headers with a zero biBitCount, which allows remote attackers to cause a denial of service (memory allocation failure) in the opj_image_create function in lib/openjp2/image.c, related to the opj_aligned_alloc_n function in opj_malloc.c.

Project: openjpeg
File: convertbmp.c

static void bmp24toimage(const OPJ_UINT8* pData, OPJ_UINT32 stride,
                         opj_image_t* image)
{
    int index;
    OPJ_UINT32 width, height;
    OPJ_UINT32 x, y;
    const OPJ_UINT8 *pSrc = NULL;

    width  = image->comps[0].w;
    height = image->comps[0].h;

    index = 0;
    pSrc = pData + (height - 1U) * stride;
    for (y = 0; y < height; y++) {
        for (x = 0; x < width; x++) {
            image->comps[0].data[index] = (OPJ_INT32)pSrc[3 * x + 2]; /* R */
            image->comps[1].data[index] = (OPJ_INT32)pSrc[3 * x + 1]; /* G */
            image->comps[2].data[index] = (OPJ_INT32)pSrc[3 * x + 0]; /* B */
            index++;
        }
        pSrc -= stride;
    }
}

Project: openjpeg
File: convertbmp.c

static opj_image_t* bmp8toimage(const OPJ_UINT8* pData, OPJ_UINT32 stride,
                                opj_image_t* image, OPJ_UINT8 const* const* pLUT)
{
    OPJ_UINT32 width, height;
    const OPJ_UINT8 *pSrc = NULL;

    width  = image->comps[0].w;
    height = image->comps[0].h;

    pSrc = pData + (height - 1U) * stride;
    if (image->numcomps == 1U) {
        opj_applyLUT8u_8u32s_C1R(pSrc, -(OPJ_INT32)stride, image->comps[0].data,
                                 (OPJ_INT32)width, pLUT[0], width, height);
    } else {
        OPJ_INT32* pDst[3];
        OPJ_INT32  pDstStride[3];

        pDst[0] = image->comps[0].data;
        pDst[1] = image->comps[1].data;
        pDst[2] = image->comps[2].data;
        pDstStride[0] = (OPJ_INT32)width;
        pDstStride[1] = (OPJ_INT32)width;
        pDstStride[2] = (OPJ_INT32)width;
        opj_applyLUT8u_8u32s_C1P3R(pSrc, -(OPJ_INT32)stride, pDst, pDstStride, pLUT,
                                   width, height);
    }
    return image;
}

Project: openjpeg
File: convertbmp.c

static void bmp_mask_get_shift_and_prec(OPJ_UINT32 mask, OPJ_UINT32* shift,
                                        OPJ_UINT32* prec)
{
    OPJ_UINT32 l_shift, l_prec;

    l_shift = l_prec = 0U;

    if (mask != 0U) {
        while ((mask & 1U) == 0U) {
            mask >>= 1;
            l_shift++;
        }
        while (mask & 1U) {
            mask >>= 1;
            l_prec++;
        }
    }
    *shift = l_shift;
    *prec = l_prec;
}

Project: openjpeg
File: convertbmp.c

static OPJ_BOOL bmp_read_file_header(FILE* IN, OPJ_BITMAPFILEHEADER* header)
{
    header->bfType  = (OPJ_UINT16)getc(IN);
    header->bfType |= (OPJ_UINT16)((OPJ_UINT32)getc(IN) << 8);

    if (header->bfType != 19778) {
        fprintf(stderr, "Error, not a BMP file!\n");
        return OPJ_FALSE;
    }

    /* FILE HEADER */
    /* ------------- */
    header->bfSize  = (OPJ_UINT32)getc(IN);
    header->bfSize |= (OPJ_UINT32)getc(IN) << 8;
    header->bfSize |= (OPJ_UINT32)getc(IN) << 16;
    header->bfSize |= (OPJ_UINT32)getc(IN) << 24;

    header->bfReserved1  = (OPJ_UINT16)getc(IN);
    header->bfReserved1 |= (OPJ_UINT16)((OPJ_UINT32)getc(IN) << 8);

    header->bfReserved2  = (OPJ_UINT16)getc(IN);
    header->bfReserved2 |= (OPJ_UINT16)((OPJ_UINT32)getc(IN) << 8);

    header->bfOffBits  = (OPJ_UINT32)getc(IN);
    header->bfOffBits |= (OPJ_UINT32)getc(IN) << 8;
    header->bfOffBits |= (OPJ_UINT32)getc(IN) << 16;
    header->bfOffBits |= (OPJ_UINT32)getc(IN) << 24;
    return OPJ_TRUE;
}

Project: openjpeg
File: convertbmp.c

static OPJ_BOOL bmp_read_raw_data(FILE* IN, OPJ_UINT8* pData, OPJ_UINT32 stride,
                                  OPJ_UINT32 width, OPJ_UINT32 height)
{
    OPJ_ARG_NOT_USED(width);

    if (fread(pData, sizeof(OPJ_UINT8), stride * height, IN) != (stride * height)) {
        fprintf(stderr,
                "\nError: fread return a number of element different from the expected.\n");
        return OPJ_FALSE;
    }
    return OPJ_TRUE;
}

Project: openjpeg
File: convertbmp.c

static OPJ_BOOL bmp_read_rle8_data(FILE* IN, OPJ_UINT8* pData,
                                   OPJ_UINT32 stride, OPJ_UINT32 width, OPJ_UINT32 height)
{
    OPJ_UINT32 x, y;
    OPJ_UINT8 *pix;
    const OPJ_UINT8 *beyond;

    beyond = pData + stride * height;
    pix = pData;

    x = y = 0U;
    while (y < height) {
        int c = getc(IN);

        if (c) {
            int j;
            OPJ_UINT8 c1 = (OPJ_UINT8)getc(IN);

            for (j = 0; (j < c) && (x < width) &&
                    ((OPJ_SIZE_T)pix < (OPJ_SIZE_T)beyond); j++, x++, pix++) {
                *pix = c1;
            }
        } else {
            c = getc(IN);
            if (c == 0x00) { /* EOL */
                x = 0;
                ++y;
                pix = pData + y * stride + x;
            } else if (c == 0x01) { /* EOP */
                break;
            } else if (c == 0x02) { /* MOVE by dxdy */
                c = getc(IN);
                x += (OPJ_UINT32)c;
                c = getc(IN);
                y += (OPJ_UINT32)c;
                pix = pData + y * stride + x;
            } else { /* 03 .. 255 */
                int j;
                for (j = 0; (j < c) && (x < width) &&
                        ((OPJ_SIZE_T)pix < (OPJ_SIZE_T)beyond); j++, x++, pix++) {
                    OPJ_UINT8 c1 = (OPJ_UINT8)getc(IN);
                    *pix = c1;
                }
                if ((OPJ_UINT32)c & 1U) { /* skip padding byte */
                    getc(IN);
                }
            }
        }
    }/* while() */
    return OPJ_TRUE;
}

Project: openjpeg
File: convertbmp.c

static void bmpmask16toimage(const OPJ_UINT8* pData, OPJ_UINT32 stride,
                             opj_image_t* image, OPJ_UINT32 redMask, OPJ_UINT32 greenMask,
                             OPJ_UINT32 blueMask, OPJ_UINT32 alphaMask)
{
    int index;
    OPJ_UINT32 width, height;
    OPJ_UINT32 x, y;
    const OPJ_UINT8 *pSrc = NULL;
    OPJ_BOOL hasAlpha;
    OPJ_UINT32 redShift,   redPrec;
    OPJ_UINT32 greenShift, greenPrec;
    OPJ_UINT32 blueShift,  bluePrec;
    OPJ_UINT32 alphaShift, alphaPrec;

    width  = image->comps[0].w;
    height = image->comps[0].h;

    hasAlpha = image->numcomps > 3U;

    bmp_mask_get_shift_and_prec(redMask,   &redShift,   &redPrec);
    bmp_mask_get_shift_and_prec(greenMask, &greenShift, &greenPrec);
    bmp_mask_get_shift_and_prec(blueMask,  &blueShift,  &bluePrec);
    bmp_mask_get_shift_and_prec(alphaMask, &alphaShift, &alphaPrec);

    image->comps[0].bpp = redPrec;
    image->comps[0].prec = redPrec;
    image->comps[1].bpp = greenPrec;
    image->comps[1].prec = greenPrec;
    image->comps[2].bpp = bluePrec;
    image->comps[2].prec = bluePrec;
    if (hasAlpha) {
        image->comps[3].bpp = alphaPrec;
        image->comps[3].prec = alphaPrec;
    }

    index = 0;
    pSrc = pData + (height - 1U) * stride;
    for (y = 0; y < height; y++) {
        for (x = 0; x < width; x++) {
            OPJ_UINT32 value = 0U;

            value |= ((OPJ_UINT32)pSrc[2 * x + 0]) <<  0;
            value |= ((OPJ_UINT32)pSrc[2 * x + 1]) <<  8;

            image->comps[0].data[index] = (OPJ_INT32)((value & redMask)   >>
                                          redShift);   /* R */
            image->comps[1].data[index] = (OPJ_INT32)((value & greenMask) >>
                                          greenShift); /* G */
            image->comps[2].data[index] = (OPJ_INT32)((value & blueMask)  >>
                                          blueShift);  /* B */
            if (hasAlpha) {
                image->comps[3].data[index] = (OPJ_INT32)((value & alphaMask)  >>
                                              alphaShift);  /* A */
            }
            index++;
        }
        pSrc -= stride;
    }
}

Project: openjpeg
File: convertbmp.c

opj_image_t* bmptoimage(const char *filename, opj_cparameters_t *parameters)
{
    opj_image_cmptparm_t cmptparm[4];   /* maximum of 4 components */
    OPJ_UINT8 lut_R[256], lut_G[256], lut_B[256];
    OPJ_UINT8 const* pLUT[3];
    opj_image_t * image = NULL;
    FILE *IN;
    OPJ_BITMAPFILEHEADER File_h;
    OPJ_BITMAPINFOHEADER Info_h;
    OPJ_UINT32 i, palette_len, numcmpts = 1U;
    OPJ_BOOL l_result = OPJ_FALSE;
    OPJ_UINT8* pData = NULL;
    OPJ_UINT32 stride;

    pLUT[0] = lut_R;
    pLUT[1] = lut_G;
    pLUT[2] = lut_B;

    IN = fopen(filename, "rb");
    if (!IN) {
        fprintf(stderr, "Failed to open %s for reading !!\n", filename);
        return NULL;
    }

    if (!bmp_read_file_header(IN, &File_h)) {
        fclose(IN);
        return NULL;
    }
    if (!bmp_read_info_header(IN, &Info_h)) {
        fclose(IN);
        return NULL;
    }

    /* Load palette */
    if (Info_h.biBitCount <= 8U) {
        memset(&lut_R[0], 0, sizeof(lut_R));
        memset(&lut_G[0], 0, sizeof(lut_G));
        memset(&lut_B[0], 0, sizeof(lut_B));

        palette_len = Info_h.biClrUsed;
        if ((palette_len == 0U) && (Info_h.biBitCount <= 8U)) {
            palette_len = (1U << Info_h.biBitCount);
        }
        if (palette_len > 256U) {
            palette_len = 256U;
        }
        if (palette_len > 0U) {
            OPJ_UINT8 has_color = 0U;
            for (i = 0U; i < palette_len; i++) {
                lut_B[i] = (OPJ_UINT8)getc(IN);
                lut_G[i] = (OPJ_UINT8)getc(IN);
                lut_R[i] = (OPJ_UINT8)getc(IN);
                (void)getc(IN); /* padding */
                has_color |= (lut_B[i] ^ lut_G[i]) | (lut_G[i] ^ lut_R[i]);
            }
            if (has_color) {
                numcmpts = 3U;
            }
        }
    } else {
        numcmpts = 3U;
        if ((Info_h.biCompression == 3) && (Info_h.biAlphaMask != 0U)) {
            numcmpts++;
        }
    }

    if (Info_h.biWidth == 0 || Info_h.biHeight == 0) {
        fclose(IN);
        return NULL;
    }

    if (Info_h.biBitCount > (((OPJ_UINT32) - 1) - 31) / Info_h.biWidth) {
        fclose(IN);
        return NULL;
    }
    stride = ((Info_h.biWidth * Info_h.biBitCount + 31U) / 32U) *
             4U; /* rows are aligned on 32bits */
    if (Info_h.biBitCount == 4 &&
            Info_h.biCompression == 2) { /* RLE 4 gets decoded as 8 bits data for now... */
        if (8 > (((OPJ_UINT32) - 1) - 31) / Info_h.biWidth) {
            fclose(IN);
            return NULL;
        }
        stride = ((Info_h.biWidth * 8U + 31U) / 32U) * 4U;
    }

    if (stride > ((OPJ_UINT32) - 1) / sizeof(OPJ_UINT8) / Info_h.biHeight) {
        fclose(IN);
        return NULL;
    }
    pData = (OPJ_UINT8 *) calloc(1, stride * Info_h.biHeight * sizeof(OPJ_UINT8));
    if (pData == NULL) {
        fclose(IN);
        return NULL;
    }
    /* Place the cursor at the beginning of the image information */
    fseek(IN, 0, SEEK_SET);
    fseek(IN, (long)File_h.bfOffBits, SEEK_SET);

    switch (Info_h.biCompression) {
    case 0:
    case 3:
        /* read raw data */
        l_result = bmp_read_raw_data(IN, pData, stride, Info_h.biWidth,
                                     Info_h.biHeight);
        break;
    case 1:
        /* read rle8 data */
        l_result = bmp_read_rle8_data(IN, pData, stride, Info_h.biWidth,
                                      Info_h.biHeight);
        break;
    case 2:
        /* read rle4 data */
        l_result = bmp_read_rle4_data(IN, pData, stride, Info_h.biWidth,
                                      Info_h.biHeight);
        break;
    default:
        fprintf(stderr, "Unsupported BMP compression\n");
        l_result = OPJ_FALSE;
        break;
    }
    if (!l_result) {
        free(pData);
        fclose(IN);
        return NULL;
    }

    /* create the image */
    memset(&cmptparm[0], 0, sizeof(cmptparm));
    for (i = 0; i < 4U; i++) {
        cmptparm[i].prec = 8;
        cmptparm[i].bpp  = 8;
        cmptparm[i].sgnd = 0;
        cmptparm[i].dx   = (OPJ_UINT32)parameters->subsampling_dx;
        cmptparm[i].dy   = (OPJ_UINT32)parameters->subsampling_dy;
        cmptparm[i].w    = Info_h.biWidth;
        cmptparm[i].h    = Info_h.biHeight;
    }

    image = opj_image_create(numcmpts, &cmptparm[0],
                             (numcmpts == 1U) ? OPJ_CLRSPC_GRAY : OPJ_CLRSPC_SRGB);
    if (!image) {
        fclose(IN);
        free(pData);
        return NULL;
    }
    if (numcmpts == 4U) {
        image->comps[3].alpha = 1;
    }

    /* set image offset and reference grid */
    image->x0 = (OPJ_UINT32)parameters->image_offset_x0;
    image->y0 = (OPJ_UINT32)parameters->image_offset_y0;
    image->x1 = image->x0 + (Info_h.biWidth  - 1U) * (OPJ_UINT32)
                parameters->subsampling_dx + 1U;
    image->y1 = image->y0 + (Info_h.biHeight - 1U) * (OPJ_UINT32)
                parameters->subsampling_dy + 1U;

    /* Read the data */
    if (Info_h.biBitCount == 24 && Info_h.biCompression == 0) { /*RGB */
        bmp24toimage(pData, stride, image);
    } else if (Info_h.biBitCount == 8 &&
               Info_h.biCompression == 0) { /* RGB 8bpp Indexed */
        bmp8toimage(pData, stride, image, pLUT);
    } else if (Info_h.biBitCount == 8 && Info_h.biCompression == 1) { /*RLE8*/
        bmp8toimage(pData, stride, image, pLUT);
    } else if (Info_h.biBitCount == 4 && Info_h.biCompression == 2) { /*RLE4*/
        bmp8toimage(pData, stride, image,
                    pLUT); /* RLE 4 gets decoded as 8 bits data for now */
    } else if (Info_h.biBitCount == 32 && Info_h.biCompression == 0) { /* RGBX */
        bmpmask32toimage(pData, stride, image, 0x00FF0000U, 0x0000FF00U, 0x000000FFU,
                         0x00000000U);
    } else if (Info_h.biBitCount == 32 && Info_h.biCompression == 3) { /* bitmask */
        bmpmask32toimage(pData, stride, image, Info_h.biRedMask, Info_h.biGreenMask,
                         Info_h.biBlueMask, Info_h.biAlphaMask);
    } else if (Info_h.biBitCount == 16 && Info_h.biCompression == 0) { /* RGBX */
        bmpmask16toimage(pData, stride, image, 0x7C00U, 0x03E0U, 0x001FU, 0x0000U);
    } else if (Info_h.biBitCount == 16 && Info_h.biCompression == 3) { /* bitmask */
        if ((Info_h.biRedMask == 0U) && (Info_h.biGreenMask == 0U) &&
                (Info_h.biBlueMask == 0U)) {
            Info_h.biRedMask   = 0xF800U;
            Info_h.biGreenMask = 0x07E0U;
            Info_h.biBlueMask  = 0x001FU;
        }
        bmpmask16toimage(pData, stride, image, Info_h.biRedMask, Info_h.biGreenMask,
                         Info_h.biBlueMask, Info_h.biAlphaMask);
    } else {
        opj_image_destroy(image);
        image = NULL;
        fprintf(stderr,
                "Other system than 24 bits/pixels or 8 bits (no RLE coding) is not yet implemented [%d]\n",
                Info_h.biBitCount);
    }
    free(pData);
    fclose(IN);
    return image;
}

Project: openjpeg
File: convertbmp.c

static void opj_applyLUT8u_8u32s_C1P3R(
    OPJ_UINT8 const* pSrc, OPJ_INT32 srcStride,
    OPJ_INT32* const* pDst, OPJ_INT32 const* pDstStride,
    OPJ_UINT8 const* const* pLUT,
    OPJ_UINT32 width, OPJ_UINT32 height)
{
    OPJ_UINT32 y;
    OPJ_INT32* pR = pDst[0];
    OPJ_INT32* pG = pDst[1];
    OPJ_INT32* pB = pDst[2];
    OPJ_UINT8 const* pLUT_R = pLUT[0];
    OPJ_UINT8 const* pLUT_G = pLUT[1];
    OPJ_UINT8 const* pLUT_B = pLUT[2];

    for (y = height; y != 0U; --y) {
        OPJ_UINT32 x;

        for (x = 0; x < width; x++) {
            OPJ_UINT8 idx = pSrc[x];
            pR[x] = (OPJ_INT32)pLUT_R[idx];
            pG[x] = (OPJ_INT32)pLUT_G[idx];
            pB[x] = (OPJ_INT32)pLUT_B[idx];
        }
        pSrc += srcStride;
        pR += pDstStride[0];
        pG += pDstStride[1];
        pB += pDstStride[2];
    }
}

Project: openjpeg
File: convertbmp.c

static void opj_applyLUT8u_8u32s_C1R(
    OPJ_UINT8 const* pSrc, OPJ_INT32 srcStride,
    OPJ_INT32* pDst, OPJ_INT32 dstStride,
    OPJ_UINT8 const* pLUT,
    OPJ_UINT32 width, OPJ_UINT32 height)
{
    OPJ_UINT32 y;

    for (y = height; y != 0U; --y) {
        OPJ_UINT32 x;

        for (x = 0; x < width; x++) {
            pDst[x] = (OPJ_INT32)pLUT[pSrc[x]];
        }
        pSrc += srcStride;
        pDst += dstStride;
    }
}