Commit message: cjpeg: Fix OOB read caused by malformed 8-bit BMP

... in which one or more of the color indices is out of range for the
number of palette entries.

Fix partly borrowed from jpeg-9c.  This commit also adopts Guido's
JERR_PPM_OUTOFRANGE enum value in lieu of our project-specific
JERR_PPM_TOOLARGE enum value.

Fixes #258
CVE: CVE-2018-14498
CWE: CWE-125
Vulnerability description: get_8bit_row in rdbmp.c in libjpeg-turbo through 1.5.90 and MozJPEG through 3.3.1 allows attackers to cause a denial of service (heap-based buffer over-read and application crash) via a crafted 8-bit BMP in which one or more of the color indices is out of range for the number of palette entries.

Project: libjpeg-turbo

GLOBAL(cjpeg_source_ptr)
jinit_read_ppm(j_compress_ptr cinfo)
{
  ppm_source_ptr source;

  /* Create module interface object */
  source = (ppm_source_ptr)
    (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,
                                sizeof(ppm_source_struct));
  /* Fill in method ptrs, except get_pixel_rows which start_input sets */
  source->pub.start_input = start_input_ppm;
  source->pub.finish_input = finish_input_ppm;

  return (cjpeg_source_ptr)source;
}

Project: libjpeg-turbo

METHODDEF(JDIMENSION)
get_text_gray_row(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
/* This version is for reading text-format PGM files with any maxval */
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;
  FILE *infile = source->pub.input_file;
  register JSAMPROW ptr;
  register JSAMPLE *rescale = source->rescale;
  JDIMENSION col;
  unsigned int maxval = source->maxval;

  ptr = source->pub.buffer[0];
  for (col = cinfo->image_width; col > 0; col--) {
    *ptr++ = rescale[read_pbm_integer(cinfo, infile, maxval)];
  }
  return 1;
}

Project: libjpeg-turbo

METHODDEF(JDIMENSION)
get_text_gray_rgb_row(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
/* This version is for reading text-format PGM files with any maxval and
   converting to extended RGB */
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;
  FILE *infile = source->pub.input_file;
  register JSAMPROW ptr;
  register JSAMPLE *rescale = source->rescale;
  JDIMENSION col;
  unsigned int maxval = source->maxval;
  register int rindex = rgb_red[cinfo->in_color_space];
  register int gindex = rgb_green[cinfo->in_color_space];
  register int bindex = rgb_blue[cinfo->in_color_space];
  register int aindex = alpha_index[cinfo->in_color_space];
  register int ps = rgb_pixelsize[cinfo->in_color_space];

  ptr = source->pub.buffer[0];
  if (maxval == MAXJSAMPLE) {
    if (aindex >= 0)
      GRAY_RGB_READ_LOOP(read_pbm_integer(cinfo, infile, maxval),
                         ptr[aindex] = 0xFF;)
    else
      GRAY_RGB_READ_LOOP(read_pbm_integer(cinfo, infile, maxval),)
  } else {
    if (aindex >= 0)
      GRAY_RGB_READ_LOOP(rescale[read_pbm_integer(cinfo, infile, maxval)],
                         ptr[aindex] = 0xFF;)
    else
      GRAY_RGB_READ_LOOP(rescale[read_pbm_integer(cinfo, infile, maxval)],)
  }
  return 1;
}

Project: libjpeg-turbo

METHODDEF(JDIMENSION)
get_text_rgb_row(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
/* This version is for reading text-format PPM files with any maxval */
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;
  FILE *infile = source->pub.input_file;
  register JSAMPROW ptr;
  register JSAMPLE *rescale = source->rescale;
  JDIMENSION col;
  unsigned int maxval = source->maxval;
  register int rindex = rgb_red[cinfo->in_color_space];
  register int gindex = rgb_green[cinfo->in_color_space];
  register int bindex = rgb_blue[cinfo->in_color_space];
  register int aindex = alpha_index[cinfo->in_color_space];
  register int ps = rgb_pixelsize[cinfo->in_color_space];

  ptr = source->pub.buffer[0];
  if (maxval == MAXJSAMPLE) {
    if (aindex >= 0)
      RGB_READ_LOOP(read_pbm_integer(cinfo, infile, maxval),
                    ptr[aindex] = 0xFF;)
    else
      RGB_READ_LOOP(read_pbm_integer(cinfo, infile, maxval),)
  } else {
    if (aindex >= 0)
      RGB_READ_LOOP(rescale[read_pbm_integer(cinfo, infile, maxval)],
                    ptr[aindex] = 0xFF;)
    else
      RGB_READ_LOOP(rescale[read_pbm_integer(cinfo, infile, maxval)],)
  }
  return 1;
}

Project: libjpeg-turbo

METHODDEF(JDIMENSION)
get_text_rgb_cmyk_row(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
/* This version is for reading text-format PPM files with any maxval and
   converting to CMYK */
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;
  FILE *infile = source->pub.input_file;
  register JSAMPROW ptr;
  register JSAMPLE *rescale = source->rescale;
  JDIMENSION col;
  unsigned int maxval = source->maxval;

  ptr = source->pub.buffer[0];
  if (maxval == MAXJSAMPLE) {
    for (col = cinfo->image_width; col > 0; col--) {
      JSAMPLE r = read_pbm_integer(cinfo, infile, maxval);
      JSAMPLE g = read_pbm_integer(cinfo, infile, maxval);
      JSAMPLE b = read_pbm_integer(cinfo, infile, maxval);
      rgb_to_cmyk(r, g, b, ptr, ptr + 1, ptr + 2, ptr + 3);
      ptr += 4;
    }
  } else {
    for (col = cinfo->image_width; col > 0; col--) {
      JSAMPLE r = rescale[read_pbm_integer(cinfo, infile, maxval)];
      JSAMPLE g = rescale[read_pbm_integer(cinfo, infile, maxval)];
      JSAMPLE b = rescale[read_pbm_integer(cinfo, infile, maxval)];
      rgb_to_cmyk(r, g, b, ptr, ptr + 1, ptr + 2, ptr + 3);
      ptr += 4;
    }
  }
  return 1;
}

Project: libjpeg-turbo

METHODDEF(JDIMENSION)
get_gray_rgb_row(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
/* This version is for reading raw-byte-format PGM files with any maxval
   and converting to extended RGB */
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;
  register JSAMPROW ptr;
  register U_CHAR *bufferptr;
  register JSAMPLE *rescale = source->rescale;
  JDIMENSION col;
  unsigned int maxval = source->maxval;
  register int rindex = rgb_red[cinfo->in_color_space];
  register int gindex = rgb_green[cinfo->in_color_space];
  register int bindex = rgb_blue[cinfo->in_color_space];
  register int aindex = alpha_index[cinfo->in_color_space];
  register int ps = rgb_pixelsize[cinfo->in_color_space];

  if (!ReadOK(source->pub.input_file, source->iobuffer, source->buffer_width))
    ERREXIT(cinfo, JERR_INPUT_EOF);
  ptr = source->pub.buffer[0];
  bufferptr = source->iobuffer;
  if (maxval == MAXJSAMPLE) {
    if (aindex >= 0)
      GRAY_RGB_READ_LOOP(*bufferptr++, ptr[aindex] = 0xFF;)
    else
      GRAY_RGB_READ_LOOP(*bufferptr++,)
  } else {
    if (aindex >= 0)
      GRAY_RGB_READ_LOOP(rescale[UCH(*bufferptr++)], ptr[aindex] = 0xFF;)
    else
      GRAY_RGB_READ_LOOP(rescale[UCH(*bufferptr++)],)
  }
  return 1;
}

Project: libjpeg-turbo

METHODDEF(JDIMENSION)
get_gray_cmyk_row(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
/* This version is for reading raw-byte-format PGM files with any maxval
   and converting to CMYK */
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;
  register JSAMPROW ptr;
  register U_CHAR *bufferptr;
  register JSAMPLE *rescale = source->rescale;
  JDIMENSION col;
  unsigned int maxval = source->maxval;

  if (!ReadOK(source->pub.input_file, source->iobuffer, source->buffer_width))
    ERREXIT(cinfo, JERR_INPUT_EOF);
  ptr = source->pub.buffer[0];
  bufferptr = source->iobuffer;
  if (maxval == MAXJSAMPLE) {
    for (col = cinfo->image_width; col > 0; col--) {
      JSAMPLE gray = *bufferptr++;
      rgb_to_cmyk(gray, gray, gray, ptr, ptr + 1, ptr + 2, ptr + 3);
      ptr += 4;
    }
  } else {
    for (col = cinfo->image_width; col > 0; col--) {
      JSAMPLE gray = rescale[UCH(*bufferptr++)];
      rgb_to_cmyk(gray, gray, gray, ptr, ptr + 1, ptr + 2, ptr + 3);
      ptr += 4;
    }
  }
  return 1;
}

Project: libjpeg-turbo

METHODDEF(JDIMENSION)
get_rgb_cmyk_row(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
/* This version is for reading raw-byte-format PPM files with any maxval and
   converting to CMYK */
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;
  register JSAMPROW ptr;
  register U_CHAR *bufferptr;
  register JSAMPLE *rescale = source->rescale;
  JDIMENSION col;
  unsigned int maxval = source->maxval;

  if (!ReadOK(source->pub.input_file, source->iobuffer, source->buffer_width))
    ERREXIT(cinfo, JERR_INPUT_EOF);
  ptr = source->pub.buffer[0];
  bufferptr = source->iobuffer;
  if (maxval == MAXJSAMPLE) {
    for (col = cinfo->image_width; col > 0; col--) {
      JSAMPLE r = *bufferptr++;
      JSAMPLE g = *bufferptr++;
      JSAMPLE b = *bufferptr++;
      rgb_to_cmyk(r, g, b, ptr, ptr + 1, ptr + 2, ptr + 3);
      ptr += 4;
    }
  } else {
    for (col = cinfo->image_width; col > 0; col--) {
      JSAMPLE r = rescale[UCH(*bufferptr++)];
      JSAMPLE g = rescale[UCH(*bufferptr++)];
      JSAMPLE b = rescale[UCH(*bufferptr++)];
      rgb_to_cmyk(r, g, b, ptr, ptr + 1, ptr + 2, ptr + 3);
      ptr += 4;
    }
  }
  return 1;
}

Project: libjpeg-turbo

METHODDEF(JDIMENSION)
get_raw_row(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
/* This version is for reading raw-byte-format files with maxval = MAXJSAMPLE.
 * In this case we just read right into the JSAMPLE buffer!
 * Note that same code works for PPM and PGM files.
 */
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;

  if (!ReadOK(source->pub.input_file, source->iobuffer, source->buffer_width))
    ERREXIT(cinfo, JERR_INPUT_EOF);
  return 1;
}

Project: libjpeg-turbo
File: rdppm.c

pbm_getc(FILE *infile)
/* Read next char, skipping over any comments */
/* A comment/newline sequence is returned as a newline */
{
  register int ch;

  ch = getc(infile);
  if (ch == '#') {
    do {
      ch = getc(infile);
    } while (ch != '\n' && ch != EOF);
  }
  return ch;
}

Project: libjpeg-turbo
File: rdppm.c

start_input_ppm(j_compress_ptr cinfo, cjpeg_source_ptr sinfo)
{
  ppm_source_ptr source = (ppm_source_ptr)sinfo;
  int c;
  unsigned int w, h, maxval;
  boolean need_iobuffer, use_raw_buffer, need_rescale;

  if (getc(source->pub.input_file) != 'P')
    ERREXIT(cinfo, JERR_PPM_NOT);

  c = getc(source->pub.input_file); /* subformat discriminator character */

  /* detect unsupported variants (ie, PBM) before trying to read header */
  switch (c) {
  case '2':                     /* it's a text-format PGM file */
  case '3':                     /* it's a text-format PPM file */
  case '5':                     /* it's a raw-format PGM file */
  case '6':                     /* it's a raw-format PPM file */
    break;
  default:
    ERREXIT(cinfo, JERR_PPM_NOT);
    break;
  }

  /* fetch the remaining header info */
  w = read_pbm_integer(cinfo, source->pub.input_file, 65535);
  h = read_pbm_integer(cinfo, source->pub.input_file, 65535);
  maxval = read_pbm_integer(cinfo, source->pub.input_file, 65535);

  if (w <= 0 || h <= 0 || maxval <= 0) /* error check */
    ERREXIT(cinfo, JERR_PPM_NOT);

  cinfo->data_precision = BITS_IN_JSAMPLE; /* we always rescale data to this */
  cinfo->image_width = (JDIMENSION)w;
  cinfo->image_height = (JDIMENSION)h;
  source->maxval = maxval;

  /* initialize flags to most common settings */
  need_iobuffer = TRUE;         /* do we need an I/O buffer? */
  use_raw_buffer = FALSE;       /* do we map input buffer onto I/O buffer? */
  need_rescale = TRUE;          /* do we need a rescale array? */

  switch (c) {
  case '2':                     /* it's a text-format PGM file */
    if (cinfo->in_color_space == JCS_UNKNOWN)
      cinfo->in_color_space = JCS_GRAYSCALE;
    TRACEMS2(cinfo, 1, JTRC_PGM_TEXT, w, h);
    if (cinfo->in_color_space == JCS_GRAYSCALE)
      source->pub.get_pixel_rows = get_text_gray_row;
    else if (IsExtRGB(cinfo->in_color_space))
      source->pub.get_pixel_rows = get_text_gray_rgb_row;
    else if (cinfo->in_color_space == JCS_CMYK)
      source->pub.get_pixel_rows = get_text_gray_cmyk_row;
    else
      ERREXIT(cinfo, JERR_BAD_IN_COLORSPACE);
    need_iobuffer = FALSE;
    break;

  case '3':                     /* it's a text-format PPM file */
    if (cinfo->in_color_space == JCS_UNKNOWN)
      cinfo->in_color_space = JCS_EXT_RGB;
    TRACEMS2(cinfo, 1, JTRC_PPM_TEXT, w, h);
    if (IsExtRGB(cinfo->in_color_space))
      source->pub.get_pixel_rows = get_text_rgb_row;
    else if (cinfo->in_color_space == JCS_CMYK)
      source->pub.get_pixel_rows = get_text_rgb_cmyk_row;
    else
      ERREXIT(cinfo, JERR_BAD_IN_COLORSPACE);
    need_iobuffer = FALSE;
    break;

  case '5':                     /* it's a raw-format PGM file */
    if (cinfo->in_color_space == JCS_UNKNOWN)
      cinfo->in_color_space = JCS_GRAYSCALE;
    TRACEMS2(cinfo, 1, JTRC_PGM, w, h);
    if (maxval > 255) {
      source->pub.get_pixel_rows = get_word_gray_row;
    } else if (maxval == MAXJSAMPLE && sizeof(JSAMPLE) == sizeof(U_CHAR) &&
               cinfo->in_color_space == JCS_GRAYSCALE) {
      source->pub.get_pixel_rows = get_raw_row;
      use_raw_buffer = TRUE;
      need_rescale = FALSE;
    } else {
      if (cinfo->in_color_space == JCS_GRAYSCALE)
        source->pub.get_pixel_rows = get_scaled_gray_row;
      else if (IsExtRGB(cinfo->in_color_space))
        source->pub.get_pixel_rows = get_gray_rgb_row;
      else if (cinfo->in_color_space == JCS_CMYK)
        source->pub.get_pixel_rows = get_gray_cmyk_row;
      else
        ERREXIT(cinfo, JERR_BAD_IN_COLORSPACE);
    }
    break;

  case '6':                     /* it's a raw-format PPM file */
    if (cinfo->in_color_space == JCS_UNKNOWN)
      cinfo->in_color_space = JCS_EXT_RGB;
    TRACEMS2(cinfo, 1, JTRC_PPM, w, h);
    if (maxval > 255) {
      source->pub.get_pixel_rows = get_word_rgb_row;
    } else if (maxval == MAXJSAMPLE && sizeof(JSAMPLE) == sizeof(U_CHAR) &&
               (cinfo->in_color_space == JCS_EXT_RGB
#if RGB_RED == 0 && RGB_GREEN == 1 && RGB_BLUE == 2 && RGB_PIXELSIZE == 3
                || cinfo->in_color_space == JCS_RGB
#endif
               )) {
      source->pub.get_pixel_rows = get_raw_row;
      use_raw_buffer = TRUE;
      need_rescale = FALSE;
    } else {
      if (IsExtRGB(cinfo->in_color_space))
        source->pub.get_pixel_rows = get_rgb_row;
      else if (cinfo->in_color_space == JCS_CMYK)
        source->pub.get_pixel_rows = get_rgb_cmyk_row;
      else
        ERREXIT(cinfo, JERR_BAD_IN_COLORSPACE);
    }
    break;
  }

  if (IsExtRGB(cinfo->in_color_space))
    cinfo->input_components = rgb_pixelsize[cinfo->in_color_space];
  else if (cinfo->in_color_space == JCS_GRAYSCALE)
    cinfo->input_components = 1;
  else if (cinfo->in_color_space == JCS_CMYK)
    cinfo->input_components = 4;

  /* Allocate space for I/O buffer: 1 or 3 bytes or words/pixel. */
  if (need_iobuffer) {
    if (c == '6')
      source->buffer_width = (size_t)w * 3 *
        ((maxval <= 255) ? sizeof(U_CHAR) : (2 * sizeof(U_CHAR)));
    else
      source->buffer_width = (size_t)w *
        ((maxval <= 255) ? sizeof(U_CHAR) : (2 * sizeof(U_CHAR)));
    source->iobuffer = (U_CHAR *)
      (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,
                                  source->buffer_width);
  }

  /* Create compressor input buffer. */
  if (use_raw_buffer) {
    /* For unscaled raw-input case, we can just map it onto the I/O buffer. */
    /* Synthesize a JSAMPARRAY pointer structure */
    source->pixrow = (JSAMPROW)source->iobuffer;
    source->pub.buffer = &source->pixrow;
    source->pub.buffer_height = 1;
  } else {
    /* Need to translate anyway, so make a separate sample buffer. */
    source->pub.buffer = (*cinfo->mem->alloc_sarray)
      ((j_common_ptr)cinfo, JPOOL_IMAGE,
       (JDIMENSION)w * cinfo->input_components, (JDIMENSION)1);
    source->pub.buffer_height = 1;
  }

  /* Compute the rescaling array if required. */
  if (need_rescale) {
    long val, half_maxval;

    /* On 16-bit-int machines we have to be careful of maxval = 65535 */
    source->rescale = (JSAMPLE *)
      (*cinfo->mem->alloc_small) ((j_common_ptr)cinfo, JPOOL_IMAGE,
                                  (size_t)(((long)maxval + 1L) *
                                           sizeof(JSAMPLE)));
    half_maxval = maxval / 2;
    for (val = 0; val <= (long)maxval; val++) {
      /* The multiplication here must be done in 32 bits to avoid overflow */
      source->rescale[val] = (JSAMPLE)((val * MAXJSAMPLE + half_maxval) /
                                        maxval);
    }
  }
}