Commit message: Bug 701819: fixed ordering in if expression to avoid out-of-bounds access.

Fixes:
    ./sanbin/gs -dBATCH -dNOPAUSE -r965 -sOutputFile=tmp -sDEVICE=pcx16 ../bug-701819.pdf
CVE: CVE-2020-16305
CWE: CWE-787
Vulnerability description: A buffer overflow vulnerability in pcx_write_rle() in contrib/japanese/gdev10v.c of Artifex Software GhostScript v9.50 allows a remote attacker to cause a denial of service via a crafted PDF file. This is fixed in v9.51.

Project: ghostpdl
File: gdevpcx.c

pcx256_print_page(gx_device_printer * pdev, gp_file * file)
{
    pcx_header header;
    int code;

    header = pcx_header_prototype;
    header.version = version_3_0;
    header.bpp = 8;
    header.nplanes = 1;
    assign_ushort(header.palinfo,
                  (pdev->color_info.num_components > 1 ?
                   palinfo_color : palinfo_gray));
    code = pcx_write_page(pdev, file, &header, false);
    if (code >= 0) {		/* Write out the palette. */
        gp_fputc(0x0c, file);
        code = pc_write_palette((gx_device *) pdev, 256, file);
    }
    return code;
}

Project: ghostpdl
File: gdevpcx.c

pcx_write_page(gx_device_printer * pdev, gp_file * file, pcx_header * phdr,
               bool planar)
{
    int raster = gdev_prn_raster(pdev);
    uint rsize = ROUND_UP((pdev->width * phdr->bpp + 7) >> 3, 2);	/* PCX format requires even */
    int height = pdev->height;
    int depth = pdev->color_info.depth;
    uint lsize = raster + rsize;
    byte *line = gs_alloc_bytes(pdev->memory, lsize, "pcx file buffer");
    byte *plane = line + raster;
    int y;
    int code = 0;		/* return code */

    if (line == 0)		/* can't allocate line buffer */
        return_error(gs_error_VMerror);

    /* Fill in the other variable entries in the header struct. */

    assign_ushort(phdr->x2, pdev->width - 1);
    assign_ushort(phdr->y2, height - 1);
    assign_ushort(phdr->hres, (int)pdev->x_pixels_per_inch);
    assign_ushort(phdr->vres, (int)pdev->y_pixels_per_inch);
    assign_ushort(phdr->bpl, (planar || depth == 1 ? rsize :
                              raster + (raster & 1)));

    /* Write the header. */

    if (gp_fwrite((const char *)phdr, 1, 128, file) < 128) {
        code = gs_error_ioerror;
        goto pcx_done;
    }
    /* Write the contents of the image. */
    for (y = 0; y < height; y++) {
        byte *row;
        byte *end;

        code = gdev_prn_get_bits(pdev, y, line, &row);
        if (code < 0)
            break;
        end = row + raster;
        if (!planar) {		/* Just write the bits. */
            if (raster & 1) {	/* Round to even, with predictable padding. */
                *end = end[-1];
                ++end;
            }
            pcx_write_rle(row, end, 1, file);
        } else
            switch (depth) {

                case 4:
                    {
                        byte *pend = plane + rsize;
                        int shift;

                        for (shift = 0; shift < 4; shift++) {
                            register byte *from, *to;
                            register int bright = 1 << shift;
                            register int bleft = bright << 4;

                            for (from = row, to = plane;
                                 from < end; from += 4
                                ) {
                                *to++ =
                                    (from[0] & bleft ? 0x80 : 0) |
                                    (from[0] & bright ? 0x40 : 0) |
                                    (from[1] & bleft ? 0x20 : 0) |
                                    (from[1] & bright ? 0x10 : 0) |
                                    (from[2] & bleft ? 0x08 : 0) |
                                    (from[2] & bright ? 0x04 : 0) |
                                    (from[3] & bleft ? 0x02 : 0) |
                                    (from[3] & bright ? 0x01 : 0);
                            }
                            /* We might be one byte short of rsize. */
                            if (to < pend)
                                *to = to[-1];
                            pcx_write_rle(plane, pend, 1, file);
                        }
                    }
                    break;

                case 24:
                    {
                        int pnum;

                        for (pnum = 0; pnum < 3; ++pnum) {
                            pcx_write_rle(row + pnum, row + raster, 3, file);
                            if (pdev->width & 1)
                                gp_fputc(0, file);		/* pad to even */
                        }
                    }
                    break;

                default:
                    code = gs_note_error(gs_error_rangecheck);
                    goto pcx_done;

            }
    }

  pcx_done:
    gs_free_object(pdev->memory, line, "pcx file buffer");

    return code;
}

Project: ghostpdl
File: gdevpcx.c

pcx_write_rle(const byte * from, const byte * end, int step, gp_file * file)
{				/*
                                 * The PCX format theoretically allows encoding runs of 63
                                 * identical bytes, but some readers can't handle repetition
                                 * counts greater than 15.
                                 */
#define MAX_RUN_COUNT 15
    int max_run = step * MAX_RUN_COUNT;

    while (from < end) {
        byte data = *from;

        from += step;
        if (from >= end || data != *from) {
            if (data >= 0xc0)
                gp_fputc(0xc1, file);
        } else {
            const byte *start = from;

            while ((from < end) && (*from == data))
                from += step;
            /* Now (from - start) / step + 1 is the run length. */
            while (from - start >= max_run) {
                gp_fputc(0xc0 + MAX_RUN_COUNT, file);
                gp_fputc(data, file);
                start += max_run;
            }
            if (from > start || data >= 0xc0)
                gp_fputc((from - start) / step + 0xc1, file);
        }
        gp_fputc(data, file);
    }
#undef MAX_RUN_COUNT
}

Project: ghostpdl
File: gdevpcx.c

pcx16_print_page(gx_device_printer * pdev, gp_file * file)
{
    pcx_header header;

    header = pcx_header_prototype;
    header.version = version_2_8_with_palette;
    header.bpp = 1;
    header.nplanes = 4;
    /* Fill the EGA palette appropriately. */
    memcpy((byte *) header.palette, pcx_ega_palette,
           sizeof(pcx_ega_palette));
    return pcx_write_page(pdev, file, &header, true);
}

Project: ghostpdl
File: gdevpcx.c

pcxcmyk_print_page(gx_device_printer * pdev, gp_file * file)
{
    pcx_header header;

    header = pcx_header_prototype;
    header.version = 2;
    header.bpp = 4;
    header.nplanes = 1;
    /* Fill the palette appropriately. */
    memcpy((byte *) header.palette, pcx_cmyk_palette,
           sizeof(pcx_cmyk_palette));
    return pcx_write_page(pdev, file, &header, false);
}

Project: ghostpdl
File: gdevpcx.c

pcx24b_print_page(gx_device_printer * pdev, gp_file * file)
{
    pcx_header header;

    header = pcx_header_prototype;
    header.version = version_3_0;
    header.bpp = 8;
    header.nplanes = 3;
    assign_ushort(header.palinfo, palinfo_color);
    return pcx_write_page(pdev, file, &header, true);
}

Project: ghostpdl
File: gdevpcx.c

pcxmono_print_page(gx_device_printer * pdev, gp_file * file)
{
    pcx_header header;

    header = pcx_header_prototype;
    header.version = version_2_8_with_palette;
    header.bpp = 1;
    header.nplanes = 1;
    assign_ushort(header.palinfo, palinfo_gray);
    /* Set the first two entries of the short palette. */
    memcpy((byte *) header.palette, "\000\000\000\377\377\377", 6);
    return pcx_write_page(pdev, file, &header, false);
}