Commit message: Make ZlibInStream more robust against failures

Move the checks around to avoid missing cases where we might access
memory that is no longer valid. Also avoid touching the underlying
stream implicitly (e.g. via the destructor) as it might also no
longer be valid.

A malicious server could theoretically use this for remote code
execution in the client.

Issue found by Pavel Cheremushkin from Kaspersky Lab
CVE: CVE-2019-15691
CWE: CWE-672
Vulnerability description: TigerVNC version prior to 1.10.1 is vulnerable to stack use-after-return, which occurs due to incorrect usage of stack memory in ZRLEDecoder. If decoding routine would throw an exception, ZRLEDecoder may try to access stack variable, which has been already freed during the process of stack unwinding. Exploitation of this vulnerability could potentially result into remote code execution. This attack appear to be exploitable via network connectivity.

Project: tigervnc

void ZRLE_DECODE (const Rect& r, rdr::InStream* is,
                  rdr::ZlibInStream* zis,
                  const PixelFormat& pf, ModifiablePixelBuffer* pb)
{
  int length = is->readU32();
  zis->setUnderlying(is, length);
  Rect t;
  PIXEL_T buf[64 * 64];

  for (t.tl.y = r.tl.y; t.tl.y < r.br.y; t.tl.y += 64) {

    t.br.y = __rfbmin(r.br.y, t.tl.y + 64);

    for (t.tl.x = r.tl.x; t.tl.x < r.br.x; t.tl.x += 64) {

      t.br.x = __rfbmin(r.br.x, t.tl.x + 64);

      int mode = zis->readU8();
      bool rle = mode & 128;
      int palSize = mode & 127;
      PIXEL_T palette[128];

      for (int i = 0; i < palSize; i++) {
        palette[i] = READ_PIXEL(zis);
      }

      if (palSize == 1) {
        PIXEL_T pix = palette[0];
        pb->fillRect(pf, t, &pix);
        continue;
      }

      if (!rle) {
        if (palSize == 0) {

          // raw

#ifdef CPIXEL
          for (PIXEL_T* ptr = buf; ptr < buf+t.area(); ptr++) {
            *ptr = READ_PIXEL(zis);
          }
#else
          zis->readBytes(buf, t.area() * (BPP / 8));
#endif

        } else {

          // packed pixels
          int bppp = ((palSize > 16) ? 8 :
                      ((palSize > 4) ? 4 : ((palSize > 2) ? 2 : 1)));

          PIXEL_T* ptr = buf;

          for (int i = 0; i < t.height(); i++) {
            PIXEL_T* eol = ptr + t.width();
            rdr::U8 byte = 0;
            rdr::U8 nbits = 0;

            while (ptr < eol) {
              if (nbits == 0) {
                byte = zis->readU8();
                nbits = 8;
              }
              nbits -= bppp;
              rdr::U8 index = (byte >> nbits) & ((1 << bppp) - 1) & 127;
              *ptr++ = palette[index];
            }
          }
        }

      } else {

        if (palSize == 0) {

          // plain RLE

          PIXEL_T* ptr = buf;
          PIXEL_T* end = ptr + t.area();
          while (ptr < end) {
            PIXEL_T pix = READ_PIXEL(zis);
            int len = 1;
            int b;
            do {
              b = zis->readU8();
              len += b;
            } while (b == 255);

            if (end - ptr < len) {
              throw Exception ("ZRLE decode error");
            }

            while (len-- > 0) *ptr++ = pix;

          }
        } else {

          // palette RLE

          PIXEL_T* ptr = buf;
          PIXEL_T* end = ptr + t.area();
          while (ptr < end) {
            int index = zis->readU8();
            int len = 1;
            if (index & 128) {
              int b;
              do {
                b = zis->readU8();
                len += b;
              } while (b == 255);

              if (end - ptr < len) {
                throw Exception ("ZRLE decode error");
              }
            }

            index &= 127;

            PIXEL_T pix = palette[index];

            while (len-- > 0) *ptr++ = pix;
          }
        }
      }

      pb->imageRect(pf, t, buf);
    }
  }

  zis->flushUnderlying();
  zis->setUnderlying(NULL, 0);
}