Commit message: Bug 701807: avoid buffer overflow in tiff12_print_page().

Fixes:
    ./sanbin/gs -r650 -sOutputFile=tmp -sDEVICE=tiff12nc ../bug-701807.pdf
CVE: CVE-2020-16300
CWE: CWE-787
Vulnerability description: A buffer overflow vulnerability in tiff12_print_page() in devices/gdevtfnx.c of Artifex Software GhostScript v9.50 allows a remote attacker to cause a denial of service via a crafted PDF file. This is fixed in v9.51.

Project: ghostpdl
File: gdevtfnx.c

tiff_rgb_print_page(gx_device_printer * pdev, gp_file * file)
{
    gx_device_tiff *const tfdev = (gx_device_tiff *)pdev;
    int code;

    /* open the TIFF device */
    if (gdev_prn_file_is_new(pdev)) {
        tfdev->tif = tiff_from_filep(pdev, pdev->dname, file, tfdev->BigEndian, tfdev->UseBigTIFF);
        if (!tfdev->tif)
            return_error(gs_error_invalidfileaccess);
    }

    code = gdev_tiff_begin_page(tfdev, file);
    if (code < 0)
        return code;

    TIFFSetField(tfdev->tif, TIFFTAG_BITSPERSAMPLE,
                 pdev->color_info.depth / pdev->color_info.num_components);
    tiff_set_rgb_fields(tfdev);

    /* Write the page data. */
    return tiff_print_page(pdev, tfdev->tif, 0);
}

Project: ghostpdl
File: gdevtfnx.c

tiff_set_rgb_fields(gx_device_tiff *tfdev)
{
    /* Put in a switch statement in case we want to have others */
    switch (tfdev->icc_struct->device_profile[0]->data_cs) {
        case gsRGB:
            TIFFSetField(tfdev->tif, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_RGB);
            break;
        case gsCIELAB:
            TIFFSetField(tfdev->tif, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_ICCLAB);
            break;
        default:
            TIFFSetField(tfdev->tif, TIFFTAG_PHOTOMETRIC, PHOTOMETRIC_RGB);
            break;
    }
    TIFFSetField(tfdev->tif, TIFFTAG_FILLORDER, FILLORDER_MSB2LSB);
    TIFFSetField(tfdev->tif, TIFFTAG_SAMPLESPERPIXEL, 3);

    tiff_set_compression((gx_device_printer *)tfdev, tfdev->tif,
                         tfdev->Compression, tfdev->MaxStripSize);
}

Project: ghostpdl
File: gdevtfnx.c

tiff12_print_page(gx_device_printer * pdev, gp_file * file)
{
    gx_device_tiff *const tfdev = (gx_device_tiff *)pdev;
    int code;

    /* open the TIFF device */
    if (gdev_prn_file_is_new(pdev)) {
        tfdev->tif = tiff_from_filep(pdev, pdev->dname, file, tfdev->BigEndian, tfdev->UseBigTIFF);
        if (!tfdev->tif)
            return_error(gs_error_invalidfileaccess);
    }

    code = gdev_tiff_begin_page(tfdev, file);
    if (code < 0)
        return code;

    TIFFSetField(tfdev->tif, TIFFTAG_BITSPERSAMPLE, 4);
    tiff_set_rgb_fields(tfdev);

    TIFFCheckpointDirectory(tfdev->tif);

    /* Write the page data. */
    {
        int y;
        int size = gdev_prn_raster(pdev);

        /* We allocate an extra 5 bytes to avoid buffer overflow when accessing
        src[5] below, if size if not multiple of 6. This fixes bug-701807. */
        int size_alloc = size + 5;
        byte *data = gs_alloc_bytes(pdev->memory, size_alloc, "tiff12_print_page");

        if (data == 0)
            return_error(gs_error_VMerror);

        memset(data, 0, size_alloc);

        for (y = 0; y < pdev->height; ++y) {
            const byte *src;
            byte *dest;
            int x;

            code = gdev_prn_copy_scan_lines(pdev, y, data, size);
            if (code < 0)
                break;

            for (src = data, dest = data, x = 0; x < size;
                 src += 6, dest += 3, x += 6
                ) {
                dest[0] = (src[0] & 0xf0) | (src[1] >> 4);
                dest[1] = (src[2] & 0xf0) | (src[3] >> 4);
                dest[2] = (src[4] & 0xf0) | (src[5] >> 4);
            }
            TIFFWriteScanline(tfdev->tif, data, y, 0);
        }
        gs_free_object(pdev->memory, data, "tiff12_print_page");

        TIFFWriteDirectory(tfdev->tif);
    }

    return code;
}