Commit message: Bug 701792: Avoid going beyond buffer in GetNumSameData() and GetNumWrongData().

GetNumSameData() compared buffer contents before checking that we are still
within bounds of buffer, which caused the bug.

Have made similar fix to GetNumWrongData() because it has similar error.

Fixes address sanitizer error in:
    ./sanbin/gs -sOutputFile=tmp -sDEVICE=lips4v ../bug-701792.pdf
CVE: CVE-2020-16296
CWE: CWE-787
Vulnerability description: A buffer overflow vulnerability in GetNumWrongData() in contrib/lips4/gdevlips.c of Artifex Software GhostScript v9.50 allows a remote attacker to cause a denial of service via a crafted PDF file. This is fixed in v9.51.

Project: ghostpdl
File: gdevlips.c

lips_rle_encode(byte * inBuff, byte * outBuff, int Length)
{
    int i = 0;
    byte value;
    int count = 0;
    byte *ptr = inBuff;

    value = *ptr;
    ptr++;

    while (ptr < inBuff + Length) {
        if (*ptr == value) {
            count++;
            if (count > RLECOUNTMAX) {
                *outBuff++ = RLECOUNTMAX;
                *outBuff++ = value;
                i += 2;
                count = 0;
            }
        } else {
            *outBuff++ = count;
            *outBuff++ = value;
            i += 2;
            count = 0;
            value = *ptr;
        }
        ptr++;
    }
    *outBuff++ = count;
    *outBuff++ = value;
    i += 2;

    return i;
}

Project: ghostpdl
File: gdevlips.c

GetNumSameData(const byte * curPtr, const int maxnum)
{
    int count = 1;

    if (1 == maxnum) {
        return (1);
    }
    while (maxnum > count && *curPtr == *(curPtr + count)) {
        count++;
    }

    return (count);
}

Project: ghostpdl
File: gdevlips.c

lips_packbits_encode(byte * inBuff, byte * outBuff, int Length)
{
    int size = 0;

    while (Length) {
        int count;

        if (1 < (count = GetNumSameData(inBuff,
                                        Length > 128 ? 128 : Length))) {
            Length -= count;
            size += 2;

            *outBuff++ = -(count - 1);
            *outBuff++ = *inBuff;
            inBuff += count;
        } else {
            count = GetNumWrongData(inBuff, Length > 128 ? 128 : Length);
            Length -= count;
            size += count + 1;

            *outBuff++ = count - 1;
            while (count--) {
                *outBuff++ = *inBuff++;
            }
        }
    }

    return (size);
}

Project: ghostpdl
File: gdevlips.c

GetNumWrongData(const byte * curPtr, const int maxnum)
{
    int count = 0;

    if (1 == maxnum) {
        return (1);
    }
    while (maxnum > count+1 && *(curPtr + count) != *(curPtr + count + 1)) {
        count++;
    }

    return (count);
}

Project: ghostpdl
File: gdevlips.c

lips_mode3format_encode(byte * inBuff, byte * outBuff, int Length)
{
    int size = 0;

    while (Length) {
        int count;

        if (1 < (count = GetNumSameData(inBuff,
                                        Length > 257 ? 257 : Length))) {
            Length -= count;
            size += 3;

            *outBuff++ = *inBuff;
            *outBuff++ = *inBuff;
            *outBuff++ = count - 2;
            inBuff += count;
        } else {
            count = GetNumWrongData(inBuff, Length);
            Length -= count;
            size += count;

            while (count--) {
                *outBuff++ = *inBuff++;
            }
        }
    }

    return (size);
}

Project: ghostpdl
File: gdevlips.c

lips_media_selection(int width, int height)
{
    int landscape = 0;
    int tmp;
    paper_table *pt;

    if (width > height) {
        landscape = 1;
        tmp = width;
        width = height;
        height = tmp;
    }
    for (pt = lips_paper_table; pt->num_unit < 80; pt++)
        if (pt->width == width && pt->height == height)
            break;

    return pt->num_unit + landscape;
}