Commit message: Added safety check for Compressed RTF Streams

You could potentially overflow the input pointer.

in response to #34
CVE: CVE-2017-6802
CWE: CWE-22
Vulnerability description: An issue was discovered in ytnef before 1.9.2. There is a potential heap-based buffer over-read on incoming Compressed RTF Streams, related to DecompressRTF() in libytnef.

Project: ytnef
File: ytnef.c

BYTE *DecompressRTF(variableLength *p, int *size) {
  BYTE *dst; // destination for uncompressed bytes
  BYTE *src;
  unsigned int in;
  unsigned int out;
  variableLength comp_Prebuf;
  ULONG compressedSize, uncompressedSize, magic;

  comp_Prebuf.size = strlen(RTF_PREBUF);
  comp_Prebuf.data = calloc(comp_Prebuf.size+1, 1);
  ALLOCCHECK_CHAR(comp_Prebuf.data);
  memcpy(comp_Prebuf.data, RTF_PREBUF, comp_Prebuf.size);

  src = p->data;
  in = 0;

  if (p->size < 20) {
    printf("File too small\n");
    return(NULL);
  }
  compressedSize = (ULONG)SwapDWord((BYTE*)src + in, 4);
  in += 4;
  uncompressedSize = (ULONG)SwapDWord((BYTE*)src + in, 4);
  in += 4;
  magic = SwapDWord((BYTE*)src + in, 4);
  in += 4;
  in += 4;

  // check size excluding the size field itself
  if (compressedSize != p->size - 4) {
    printf(" Size Mismatch: %u != %i\n", compressedSize, p->size - 4);
    free(comp_Prebuf.data);
    return NULL;
  }

  // process the data
  if (magic == 0x414c454d) {
    // magic number that identifies the stream as a uncompressed stream
    dst = calloc(uncompressedSize, 1);
    ALLOCCHECK_CHAR(dst);
    memcpy(dst, src + 4, uncompressedSize);
  } else if (magic == 0x75465a4c) {
    // magic number that identifies the stream as a compressed stream
    int flagCount = 0;
    int flags = 0;
    // Prevent overflow on 32 Bit Systems
    if (comp_Prebuf.size >= INT_MAX - uncompressedSize) {
       printf("Corrupted file\n");
       exit(-1);
    }
    dst = calloc(comp_Prebuf.size + uncompressedSize, 1);
    ALLOCCHECK_CHAR(dst);
    memcpy(dst, comp_Prebuf.data, comp_Prebuf.size);
    out = comp_Prebuf.size;
    while ((out < (comp_Prebuf.size + uncompressedSize)) && (in < p->size)) {
      // each flag byte flags 8 literals/references, 1 per bit
      flags = (flagCount++ % 8 == 0) ? src[in++] : flags >> 1;
      if ((flags & 1) == 1) { // each flag bit is 1 for reference, 0 for literal
        unsigned int offset = src[in++];
        unsigned int length = src[in++];
        unsigned int end;
        offset = (offset << 4) | (length >> 4); // the offset relative to block start
        length = (length & 0xF) + 2; // the number of bytes to copy
        // the decompression buffer is supposed to wrap around back
        // to the beginning when the end is reached. we save the
        // need for such a buffer by pointing straight into the data
        // buffer, and simulating this behaviour by modifying the
        // pointers appropriately.
        offset = (out / 4096) * 4096 + offset;
        if (offset >= out) // take from previous block
          offset -= 4096;
        // note: can't use System.arraycopy, because the referenced
        // bytes can cross through the current out position.
        end = offset + length;
        while ((offset < end) && (out < (comp_Prebuf.size + uncompressedSize))
             && (offset < (comp_Prebuf.size + uncompressedSize)))
          dst[out++] = dst[offset++];
      } else { // literal
        if ((out >= (comp_Prebuf.size + uncompressedSize)) ||
            (in >= p->size)) {
          printf("Corrupted stream\n");
          exit(-1);
        }
        dst[out++] = src[in++];
      }
    }
    // copy it back without the prebuffered data
    src = dst;
    dst = calloc(uncompressedSize, 1);
    ALLOCCHECK_CHAR(dst);
    memcpy(dst, src + comp_Prebuf.size, uncompressedSize);
    free(src);
    *size = uncompressedSize;
    free(comp_Prebuf.data);
    return dst;
  } else { // unknown magic number
    printf("Unknown compression type (magic number %x)\n", magic);
  }
  free(comp_Prebuf.data);
  return NULL;
}

Project: ytnef
File: ytnef.c

int TNEFParse(TNEFStruct *TNEF) {
  WORD key;
  DWORD type;
  DWORD size;
  DWORD signature;
  BYTE *data;
  WORD checksum, header_checksum;
  int i;

  if (TNEF->IO.ReadProc == NULL) {
    printf("ERROR: Setup incorrectly: No ReadProc\n");
    return YTNEF_INCORRECT_SETUP;
  }

  if (TNEF->IO.InitProc != NULL) {
    DEBUG(TNEF->Debug, 2, "About to initialize");
    if (TNEF->IO.InitProc(&TNEF->IO) != 0) {
      return YTNEF_CANNOT_INIT_DATA;
    }
    DEBUG(TNEF->Debug, 2, "Initialization finished");
  }

  DEBUG(TNEF->Debug, 2, "Reading Signature");
  if (TNEF->IO.ReadProc(&TNEF->IO, sizeof(DWORD), 1, &signature) < 1) {
    printf("ERROR: Error reading signature\n");
    if (TNEF->IO.CloseProc != NULL) {
      TNEF->IO.CloseProc(&TNEF->IO);
    }
    return YTNEF_ERROR_READING_DATA;
  }

  DEBUG(TNEF->Debug, 2, "Checking Signature");
  if (TNEFCheckForSignature(signature) < 0) {
    printf("ERROR: Signature does not match. Not TNEF.\n");
    if (TNEF->IO.CloseProc != NULL) {
      TNEF->IO.CloseProc(&TNEF->IO);
    }
    return YTNEF_NOT_TNEF_STREAM;
  }

  DEBUG(TNEF->Debug, 2, "Reading Key.");

  if (TNEFGetKey(TNEF, &key) < 0) {
    printf("ERROR: Unable to retrieve key.\n");
    if (TNEF->IO.CloseProc != NULL) {
      TNEF->IO.CloseProc(&TNEF->IO);
    }
    return YTNEF_NO_KEY;
  }

  DEBUG(TNEF->Debug, 2, "Starting Full Processing.");

  while (TNEFGetHeader(TNEF, &type, &size) == 0) {
    DEBUG2(TNEF->Debug, 2, "Header says type=0x%X, size=%u", type, size);
    DEBUG2(TNEF->Debug, 2, "Header says type=%u, size=%u", type, size);
    if(size == 0) {
      printf("ERROR: Field with size of 0\n");
      return YTNEF_ERROR_READING_DATA;
    }
    data = calloc(size, sizeof(BYTE));
    ALLOCCHECK(data);
    if (TNEFRawRead(TNEF, data, size, &header_checksum) < 0) {
      printf("ERROR: Unable to read data.\n");
      if (TNEF->IO.CloseProc != NULL) {
        TNEF->IO.CloseProc(&TNEF->IO);
      }
      free(data);
      return YTNEF_ERROR_READING_DATA;
    }
    if (TNEFRawRead(TNEF, (BYTE *)&checksum, 2, NULL) < 0) {
      printf("ERROR: Unable to read checksum.\n");
      if (TNEF->IO.CloseProc != NULL) {
        TNEF->IO.CloseProc(&TNEF->IO);
      }
      free(data);
      return YTNEF_ERROR_READING_DATA;
    }
    checksum = SwapWord((BYTE *)&checksum, sizeof(WORD));
    if (checksum != header_checksum) {
      printf("ERROR: Checksum mismatch. Data corruption?:\n");
      if (TNEF->IO.CloseProc != NULL) {
        TNEF->IO.CloseProc(&TNEF->IO);
      }
      free(data);
      return YTNEF_BAD_CHECKSUM;
    }
    for (i = 0; i < (sizeof(TNEFList) / sizeof(TNEFHandler)); i++) {
      if (TNEFList[i].id == type) {
        if (TNEFList[i].handler != NULL) {
          if (TNEFList[i].handler(TNEF, i, (char*)data, size) < 0) {
            free(data);
            if (TNEF->IO.CloseProc != NULL) {
              TNEF->IO.CloseProc(&TNEF->IO);
            }
            return YTNEF_ERROR_IN_HANDLER;
          } else {
            //  Found our handler and processed it.  now time to get out
            break;
          }
        } else {
          DEBUG2(TNEF->Debug, 1, "No handler for %s: %u bytes",
                 TNEFList[i].name, size);
        }
      }
    }

    free(data);
  }

  if (TNEF->IO.CloseProc != NULL) {
    TNEF->IO.CloseProc(&TNEF->IO);
  }
  return 0;

}