Commit message: Overflow checks for realloc for tiff decoding
CVE: CVE-2020-5310
CWE: CWE-787
Vulnerability description: libImaging/TiffDecode.c in Pillow before 6.2.2 has a TIFF decoding integer overflow, related to realloc.

Project: Pillow
File: TiffDecode.c

int ImagingLibTiffDecode(Imaging im, ImagingCodecState state, UINT8* buffer, Py_ssize_t bytes) {
    TIFFSTATE *clientstate = (TIFFSTATE *)state->context;
    char *filename = "tempfile.tif";
    char *mode = "r";
    TIFF *tiff;

    /* buffer is the encoded file, bytes is the length of the encoded file */
    /*     it all ends up in state->buffer, which is a uint8* from Imaging.h */

    TRACE(("in decoder: bytes %d\n", bytes));
    TRACE(("State: count %d, state %d, x %d, y %d, ystep %d\n", state->count, state->state,
           state->x, state->y, state->ystep));
    TRACE(("State: xsize %d, ysize %d, xoff %d, yoff %d \n", state->xsize, state->ysize,
           state->xoff, state->yoff));
    TRACE(("State: bits %d, bytes %d \n", state->bits, state->bytes));
    TRACE(("Buffer: %p: %c%c%c%c\n", buffer, (char)buffer[0], (char)buffer[1],(char)buffer[2], (char)buffer[3]));
    TRACE(("State->Buffer: %c%c%c%c\n", (char)state->buffer[0], (char)state->buffer[1],(char)state->buffer[2], (char)state->buffer[3]));
    TRACE(("Image: mode %s, type %d, bands: %d, xsize %d, ysize %d \n",
           im->mode, im->type, im->bands, im->xsize, im->ysize));
    TRACE(("Image: image8 %p, image32 %p, image %p, block %p \n",
           im->image8, im->image32, im->image, im->block));
    TRACE(("Image: pixelsize: %d, linesize %d \n",
           im->pixelsize, im->linesize));

    dump_state(clientstate);
    clientstate->size = bytes;
    clientstate->eof = clientstate->size;
    clientstate->loc = 0;
    clientstate->data = (tdata_t)buffer;
    clientstate->flrealloc = 0;
    dump_state(clientstate);

    TIFFSetWarningHandler(NULL);
    TIFFSetWarningHandlerExt(NULL);

    if (clientstate->fp) {
        TRACE(("Opening using fd: %d\n",clientstate->fp));
        lseek(clientstate->fp,0,SEEK_SET); // Sometimes, I get it set to the end.
        tiff = TIFFFdOpen(clientstate->fp, filename, mode);
    } else {
        TRACE(("Opening from string\n"));
        tiff = TIFFClientOpen(filename, mode,
                              (thandle_t) clientstate,
                              _tiffReadProc, _tiffWriteProc,
                              _tiffSeekProc, _tiffCloseProc, _tiffSizeProc,
                              _tiffMapProc, _tiffUnmapProc);
    }

    if (!tiff){
        TRACE(("Error, didn't get the tiff\n"));
        state->errcode = IMAGING_CODEC_BROKEN;
        return -1;
    }

    if (clientstate->ifd){
        int rv;
        uint32 ifdoffset = clientstate->ifd;
        TRACE(("reading tiff ifd %u\n", ifdoffset));
        rv = TIFFSetSubDirectory(tiff, ifdoffset);
        if (!rv){
            TRACE(("error in TIFFSetSubDirectory"));
            return -1;
        }
    }

    if (TIFFIsTiled(tiff)) {
        UINT32 x, y, tile_y, row_byte_size;
        UINT32 tile_width, tile_length, current_tile_width;
        UINT8 *new_data;

        TIFFGetField(tiff, TIFFTAG_TILEWIDTH, &tile_width);
        TIFFGetField(tiff, TIFFTAG_TILELENGTH, &tile_length);

        // We could use TIFFTileSize, but for YCbCr data it returns subsampled data size
        row_byte_size = (tile_width * state->bits + 7) / 8;

        /* overflow check for realloc */
        if (INT_MAX / row_byte_size < tile_length) {
            state->errcode = IMAGING_CODEC_MEMORY;
            TIFFClose(tiff);
            return -1;
        }
        
        state->bytes = row_byte_size * tile_length;

        /* realloc to fit whole tile */
        /* malloc check above */
        new_data = realloc (state->buffer, state->bytes);
        if (!new_data) {
            state->errcode = IMAGING_CODEC_MEMORY;
            TIFFClose(tiff);
            return -1;
        }

        state->buffer = new_data;

        TRACE(("TIFFTileSize: %d\n", state->bytes));

        for (y = state->yoff; y < state->ysize; y += tile_length) {
            for (x = state->xoff; x < state->xsize; x += tile_width) {
                if (ReadTile(tiff, x, y, (UINT32*) state->buffer) == -1) {
                    TRACE(("Decode Error, Tile at %dx%d\n", x, y));
                    state->errcode = IMAGING_CODEC_BROKEN;
                    TIFFClose(tiff);
                    return -1;
                }

                TRACE(("Read tile at %dx%d; \n\n", x, y));

                current_tile_width = min(tile_width, state->xsize - x);

                // iterate over each line in the tile and stuff data into image
                for (tile_y = 0; tile_y < min(tile_length, state->ysize - y); tile_y++) {
                    TRACE(("Writing tile data at %dx%d using tile_width: %d; \n", tile_y + y, x, current_tile_width));

                    // UINT8 * bbb = state->buffer + tile_y * row_byte_size;
                    // TRACE(("chars: %x%x%x%x\n", ((UINT8 *)bbb)[0], ((UINT8 *)bbb)[1], ((UINT8 *)bbb)[2], ((UINT8 *)bbb)[3]));

                    state->shuffle((UINT8*) im->image[tile_y + y] + x * im->pixelsize,
                       state->buffer + tile_y * row_byte_size,
                       current_tile_width
                    );
                }
            }
        }
    } else {
        UINT32 strip_row, row_byte_size;
        UINT8 *new_data;
        UINT32 rows_per_strip;
        int ret;

        ret = TIFFGetField(tiff, TIFFTAG_ROWSPERSTRIP, &rows_per_strip);
        if (ret != 1) {
            rows_per_strip = state->ysize;
        }
        TRACE(("RowsPerStrip: %u \n", rows_per_strip));

        // We could use TIFFStripSize, but for YCbCr data it returns subsampled data size
        row_byte_size = (state->xsize * state->bits + 7) / 8;

        /* overflow check for realloc */
        if (INT_MAX / row_byte_size < rows_per_strip) {
            state->errcode = IMAGING_CODEC_MEMORY;
            TIFFClose(tiff);
            return -1;
        }
        
        state->bytes = rows_per_strip * row_byte_size;

        TRACE(("StripSize: %d \n", state->bytes));

        /* realloc to fit whole strip */
        /* malloc check above */
        new_data = realloc (state->buffer, state->bytes);
        if (!new_data) {
            state->errcode = IMAGING_CODEC_MEMORY;
            TIFFClose(tiff);
            return -1;
        }

        state->buffer = new_data;

        for (; state->y < state->ysize; state->y += rows_per_strip) {
            if (ReadStrip(tiff, state->y, (UINT32 *)state->buffer) == -1) {
                TRACE(("Decode Error, strip %d\n", TIFFComputeStrip(tiff, state->y, 0)));
                state->errcode = IMAGING_CODEC_BROKEN;
                TIFFClose(tiff);
                return -1;
            }

            TRACE(("Decoded strip for row %d \n", state->y));

            // iterate over each row in the strip and stuff data into image
            for (strip_row = 0; strip_row < min(rows_per_strip, state->ysize - state->y); strip_row++) {
                TRACE(("Writing data into line %d ; \n", state->y + strip_row));

                // UINT8 * bbb = state->buffer + strip_row * (state->bytes / rows_per_strip);
                // TRACE(("chars: %x %x %x %x\n", ((UINT8 *)bbb)[0], ((UINT8 *)bbb)[1], ((UINT8 *)bbb)[2], ((UINT8 *)bbb)[3]));

                state->shuffle((UINT8*) im->image[state->y + state->yoff + strip_row] +
                               state->xoff * im->pixelsize,
                               state->buffer + strip_row * row_byte_size,
                               state->xsize);
            }
        }
    }

    TIFFClose(tiff);
    TRACE(("Done Decoding, Returning \n"));
    // Returning -1 here to force ImageFile.load to break, rather than
    // even think about looping back around.
    return -1;
}

Project: Pillow
File: TiffDecode.c

int ImagingLibTiffMergeFieldInfo(ImagingCodecState state, TIFFDataType field_type, int key, int is_var_length){
    // Refer to libtiff docs (http://www.simplesystems.org/libtiff/addingtags.html)
    TIFFSTATE *clientstate = (TIFFSTATE *)state->context;
    char field_name[10];
    uint32 n;
    int status = 0;

    // custom fields added with ImagingLibTiffMergeFieldInfo are only used for
    // decoding, ignore readcount;
    int readcount = 0;
    // we support writing a single value, or a variable number of values
    int writecount = 1;
    // whether the first value should encode the number of values.
    int passcount = 0;

    TIFFFieldInfo info[] = {
        { key, readcount, writecount, field_type, FIELD_CUSTOM, 1, passcount, field_name }
    };

    if (is_var_length) {
        info[0].field_writecount = -1;
    }

    if (is_var_length && field_type != TIFF_ASCII) {
        info[0].field_passcount = 1;
    }

    n = sizeof(info) / sizeof(info[0]);

    // Test for libtiff 4.0 or later, excluding libtiff 3.9.6 and 3.9.7
#if TIFFLIB_VERSION >= 20111221 && TIFFLIB_VERSION != 20120218 && TIFFLIB_VERSION != 20120922
    status = TIFFMergeFieldInfo(clientstate->tiff, info, n);
#else
    TIFFMergeFieldInfo(clientstate->tiff, info, n);
#endif
    return status;
}

Project: Pillow
File: TiffDecode.c

int ReadTile(TIFF* tiff, UINT32 col, UINT32 row, UINT32* buffer) {
    uint16 photometric;

    TIFFGetField(tiff, TIFFTAG_PHOTOMETRIC, &photometric);

    // To avoid dealing with YCbCr subsampling, let libtiff handle it
    if (photometric == PHOTOMETRIC_YCBCR) {
        UINT32 tile_width, tile_height, swap_line_size, i_row;
        UINT32* swap_line;

        TIFFGetField(tiff, TIFFTAG_TILEWIDTH, &tile_width);
        TIFFGetField(tiff, TIFFTAG_TILELENGTH, &tile_height);

        swap_line_size = tile_width * sizeof(UINT32);
        if (tile_width != swap_line_size / sizeof(UINT32)) {
            return -1;
        }

        /* Read the tile into an RGBA array */
        if (!TIFFReadRGBATile(tiff, col, row, buffer)) {
            return -1;
        }

        swap_line = (UINT32*)malloc(swap_line_size);
        if (swap_line == NULL) {
            return -1;
        }
        /*
         * For some reason the TIFFReadRGBATile() function chooses the
         * lower left corner as the origin.  Vertically mirror scanlines.
         */
        for(i_row = 0; i_row < tile_height / 2; i_row++) {
            UINT32 *top_line, *bottom_line;

            top_line = buffer + tile_width * i_row;
            bottom_line = buffer + tile_width * (tile_height - i_row - 1);

            memcpy(swap_line, top_line, 4*tile_width);
            memcpy(top_line, bottom_line, 4*tile_width);
            memcpy(bottom_line, swap_line, 4*tile_width);
        }

        free(swap_line);

        return 0;
    }

    if (TIFFReadTile(tiff, (tdata_t)buffer, col, row, 0, 0) == -1) {
        TRACE(("Decode Error, Tile at %dx%d\n", col, row));
        return -1;
    }

    TRACE(("Successfully read tile at %dx%d; \n\n", col, row));

    return 0;
}