Commit message: Thumbnail size range check
CVE: CVE-2020-15503
CWE: CWE-125
Vulnerability description: LibRaw before 0.20-RC1 lacks a thumbnail size range check. This affects decoders/unpack_thumb.cpp, postprocessing/mem_image.cpp, and utils/thumb_utils.cpp. For example, malloc(sizeof(libraw_processed_image_t)+T.tlength) occurs without validating T.tlength.

Project: LibRaw
File: thumb_utils.cpp

void LibRaw::kodak_thumb_loader()
{
  INT64 est_datasize =
      T.theight * T.twidth / 3; // is 0.3 bytes per pixel good estimate?
  if (ID.toffset < 0)
    throw LIBRAW_EXCEPTION_IO_CORRUPT;

  if (ID.toffset + est_datasize > ID.input->size() + THUMB_READ_BEYOND)
    throw LIBRAW_EXCEPTION_IO_EOF;

  // some kodak cameras
  ushort s_height = S.height, s_width = S.width, s_iwidth = S.iwidth,
         s_iheight = S.iheight;
  ushort s_flags = libraw_internal_data.unpacker_data.load_flags;
  libraw_internal_data.unpacker_data.load_flags = 12;
  int s_colors = P1.colors;
  unsigned s_filters = P1.filters;
  ushort(*s_image)[4] = imgdata.image;

  S.height = T.theight;
  S.width = T.twidth;
  P1.filters = 0;

  if (thumb_load_raw == &LibRaw::kodak_ycbcr_load_raw)
  {
    S.height += S.height & 1;
    S.width += S.width & 1;
  }

  imgdata.image =
      (ushort(*)[4])calloc(S.iheight * S.iwidth, sizeof(*imgdata.image));
  merror(imgdata.image, "LibRaw::kodak_thumb_loader()");

  ID.input->seek(ID.toffset, SEEK_SET);
  // read kodak thumbnail into T.image[]
  try
  {
    (this->*thumb_load_raw)();
  }
  catch (...)
  {
    free(imgdata.image);
    imgdata.image = s_image;

    T.twidth = 0;
    S.width = s_width;

    S.iwidth = s_iwidth;
    S.iheight = s_iheight;

    T.theight = 0;
    S.height = s_height;

    T.tcolors = 0;
    P1.colors = s_colors;

    P1.filters = s_filters;
    T.tlength = 0;
    libraw_internal_data.unpacker_data.load_flags = s_flags;
    return;
  }

  // from scale_colors
  {
    double dmax;
    float scale_mul[4];
    int c, val;
    for (dmax = DBL_MAX, c = 0; c < 3; c++)
      if (dmax > C.pre_mul[c])
        dmax = C.pre_mul[c];

    for (c = 0; c < 3; c++)
      scale_mul[c] = (C.pre_mul[c] / dmax) * 65535.0 / C.maximum;
    scale_mul[3] = scale_mul[1];

    size_t size = S.height * S.width;
    for (unsigned i = 0; i < size * 4; i++)
    {
      val = imgdata.image[0][i];
      if (!val)
        continue;
      val *= scale_mul[i & 3];
      imgdata.image[0][i] = CLIP(val);
    }
  }

  // from convert_to_rgb
  ushort *img;
  int row, col;

  int(*t_hist)[LIBRAW_HISTOGRAM_SIZE] =
      (int(*)[LIBRAW_HISTOGRAM_SIZE])calloc(sizeof(*t_hist), 4);
  merror(t_hist, "LibRaw::kodak_thumb_loader()");

  float out[3], out_cam[3][4] = {{2.81761312, -1.98369181, 0.166078627, 0},
                                 {-0.111855984, 1.73688626, -0.625030339, 0},
                                 {-0.0379119813, -0.891268849, 1.92918086, 0}};

  for (img = imgdata.image[0], row = 0; row < S.height; row++)
    for (col = 0; col < S.width; col++, img += 4)
    {
      out[0] = out[1] = out[2] = 0;
      int c;
      for (c = 0; c < 3; c++)
      {
        out[0] += out_cam[0][c] * img[c];
        out[1] += out_cam[1][c] * img[c];
        out[2] += out_cam[2][c] * img[c];
      }
      for (c = 0; c < 3; c++)
        img[c] = CLIP((int)out[c]);
      for (c = 0; c < P1.colors; c++)
        t_hist[c][img[c] >> 3]++;
    }

  // from gamma_lut
  int(*save_hist)[LIBRAW_HISTOGRAM_SIZE] =
      libraw_internal_data.output_data.histogram;
  libraw_internal_data.output_data.histogram = t_hist;

  // make curve output curve!
  ushort *t_curve = (ushort *)calloc(sizeof(C.curve), 1);
  merror(t_curve, "LibRaw::kodak_thumb_loader()");
  memmove(t_curve, C.curve, sizeof(C.curve));
  memset(C.curve, 0, sizeof(C.curve));
  {
    int perc, val, total, t_white = 0x2000, c;

    perc = S.width * S.height * 0.01; /* 99th percentile white level */
    if (IO.fuji_width)
      perc /= 2;
    if (!((O.highlight & ~2) || O.no_auto_bright))
      for (t_white = c = 0; c < P1.colors; c++)
      {
        for (val = 0x2000, total = 0; --val > 32;)
          if ((total += libraw_internal_data.output_data.histogram[c][val]) >
              perc)
            break;
        if (t_white < val)
          t_white = val;
      }
    gamma_curve(O.gamm[0], O.gamm[1], 2, (t_white << 3) / O.bright);
  }

  libraw_internal_data.output_data.histogram = save_hist;
  free(t_hist);

  // from write_ppm_tiff - copy pixels into bitmap

  int s_flip = imgdata.sizes.flip;
  if (imgdata.params.raw_processing_options &
      LIBRAW_PROCESSING_NO_ROTATE_FOR_KODAK_THUMBNAILS)
    imgdata.sizes.flip = 0;

  S.iheight = S.height;
  S.iwidth = S.width;
  if (S.flip & 4)
    SWAP(S.height, S.width);

  if (T.thumb)
    free(T.thumb);
  T.thumb = (char *)calloc(S.width * S.height, P1.colors);
  merror(T.thumb, "LibRaw::kodak_thumb_loader()");
  T.tlength = S.width * S.height * P1.colors;

  // from write_tiff_ppm
  {
    int soff = flip_index(0, 0);
    int cstep = flip_index(0, 1) - soff;
    int rstep = flip_index(1, 0) - flip_index(0, S.width);

    for (int row = 0; row < S.height; row++, soff += rstep)
    {
      char *ppm = T.thumb + row * S.width * P1.colors;
      for (int col = 0; col < S.width; col++, soff += cstep)
        for (int c = 0; c < P1.colors; c++)
          ppm[col * P1.colors + c] =
              imgdata.color.curve[imgdata.image[soff][c]] >> 8;
    }
  }

  memmove(C.curve, t_curve, sizeof(C.curve));
  free(t_curve);

  // restore variables
  free(imgdata.image);
  imgdata.image = s_image;

  if (imgdata.params.raw_processing_options &
      LIBRAW_PROCESSING_NO_ROTATE_FOR_KODAK_THUMBNAILS)
    imgdata.sizes.flip = s_flip;

  T.twidth = S.width;
  S.width = s_width;

  S.iwidth = s_iwidth;
  S.iheight = s_iheight;

  T.theight = S.height;
  S.height = s_height;

  T.tcolors = P1.colors;
  P1.colors = s_colors;

  P1.filters = s_filters;
  libraw_internal_data.unpacker_data.load_flags = s_flags;
}

Project: LibRaw
File: unpack_thumb.cpp

int LibRaw::unpack_thumb(void)
{
  CHECK_ORDER_LOW(LIBRAW_PROGRESS_IDENTIFY);
  CHECK_ORDER_BIT(LIBRAW_PROGRESS_THUMB_LOAD);

  try
  {
    if (!libraw_internal_data.internal_data.input)
      return LIBRAW_INPUT_CLOSED;

    int t_colors = libraw_internal_data.unpacker_data.thumb_misc >> 5 & 7;
    int t_bytesps = (libraw_internal_data.unpacker_data.thumb_misc & 31) / 8;

    if (!ID.toffset && !(imgdata.thumbnail.tlength > 0 &&
                         load_raw == &LibRaw::broadcom_load_raw) // RPi
    )
    {
      return LIBRAW_NO_THUMBNAIL;
    }
    else if (thumb_load_raw)
    {
      kodak_thumb_loader();
      T.tformat = LIBRAW_THUMBNAIL_BITMAP;
      SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
      return 0;
    }
    else
    {
#ifdef USE_X3FTOOLS
	if (write_thumb == &LibRaw::x3f_thumb_loader)
      {
        INT64 tsize = x3f_thumb_size();
        if (tsize < 2048 || INT64(ID.toffset) + tsize < 1)
          throw LIBRAW_EXCEPTION_IO_CORRUPT;

        if (INT64(ID.toffset) + tsize > ID.input->size() + THUMB_READ_BEYOND)
          throw LIBRAW_EXCEPTION_IO_EOF;
      }
#else
	if (0) {}
#endif
      else
      {
        if (INT64(ID.toffset) + INT64(T.tlength) < 1)
          throw LIBRAW_EXCEPTION_IO_CORRUPT;

        if (INT64(ID.toffset) + INT64(T.tlength) >
            ID.input->size() + THUMB_READ_BEYOND)
          throw LIBRAW_EXCEPTION_IO_EOF;
      }

      ID.input->seek(ID.toffset, SEEK_SET);
      if (write_thumb == &LibRaw::jpeg_thumb)
      {
        if (T.thumb)
          free(T.thumb);
        T.thumb = (char *)malloc(T.tlength);
        merror(T.thumb, "jpeg_thumb()");
        ID.input->read(T.thumb, 1, T.tlength);
        unsigned char *tthumb = (unsigned char *)T.thumb;
        tthumb[0] = 0xff;
        tthumb[1] = 0xd8;
#ifdef NO_JPEG
        T.tcolors = 3;
#else
        {
          jpegErrorManager jerr;
          struct jpeg_decompress_struct cinfo;
          cinfo.err = jpeg_std_error(&jerr.pub);
          jerr.pub.error_exit = jpegErrorExit;
          if (setjmp(jerr.setjmp_buffer))
          {
          err2:
            // Error in original JPEG thumb, read it again because
            // original bytes 0-1 was damaged above
            jpeg_destroy_decompress(&cinfo);
            T.tcolors = 3;
            T.tformat = LIBRAW_THUMBNAIL_UNKNOWN;
            ID.input->seek(ID.toffset, SEEK_SET);
            ID.input->read(T.thumb, 1, T.tlength);
            SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
            return 0;
          }
          jpeg_create_decompress(&cinfo);
          jpeg_mem_src(&cinfo, (unsigned char *)T.thumb, T.tlength);
          int rc = jpeg_read_header(&cinfo, TRUE);
          if (rc != 1)
            goto err2;
          T.tcolors = (cinfo.num_components > 0 && cinfo.num_components <= 3)
                          ? cinfo.num_components
                          : 3;
          jpeg_destroy_decompress(&cinfo);
        }
#endif
        T.tformat = LIBRAW_THUMBNAIL_JPEG;
        SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
        return 0;
      }
      else if (write_thumb == &LibRaw::layer_thumb)
      {
        int colors = libraw_internal_data.unpacker_data.thumb_misc >> 5 & 7;
        if (colors != 1 && colors != 3)
          return LIBRAW_UNSUPPORTED_THUMBNAIL;

        int tlength = T.twidth * T.theight;
        if (T.thumb)
          free(T.thumb);
        T.thumb = (char *)calloc(colors, tlength);
        merror(T.thumb, "layer_thumb()");
        unsigned char *tbuf = (unsigned char *)calloc(colors, tlength);
        merror(tbuf, "layer_thumb()");
        ID.input->read(tbuf, colors, T.tlength);
        if (libraw_internal_data.unpacker_data.thumb_misc >> 8 &&
            colors == 3) // GRB order
          for (int i = 0; i < tlength; i++)
          {
            T.thumb[i * 3] = tbuf[i + tlength];
            T.thumb[i * 3 + 1] = tbuf[i];
            T.thumb[i * 3 + 2] = tbuf[i + 2 * tlength];
          }
        else if (colors == 3) // RGB or 1-channel
          for (int i = 0; i < tlength; i++)
          {
            T.thumb[i * 3] = tbuf[i];
            T.thumb[i * 3 + 1] = tbuf[i + tlength];
            T.thumb[i * 3 + 2] = tbuf[i + 2 * tlength];
          }
        else if (colors == 1)
        {
          free(T.thumb);
          T.thumb = (char *)tbuf;
          tbuf = 0;
        }
        if (tbuf)
          free(tbuf);
        T.tcolors = colors;
        T.tlength = colors * tlength;
        T.tformat = LIBRAW_THUMBNAIL_BITMAP;
        SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
        return 0;
      }
      else if (write_thumb == &LibRaw::rollei_thumb)
      {
        int i;
        int tlength = T.twidth * T.theight;
        if (T.thumb)
          free(T.thumb);
        T.tcolors = 3;
        T.thumb = (char *)calloc(T.tcolors, tlength);
        merror(T.thumb, "layer_thumb()");
        unsigned short *tbuf = (unsigned short *)calloc(2, tlength);
        merror(tbuf, "layer_thumb()");
        read_shorts(tbuf, tlength);
        for (i = 0; i < tlength; i++)
        {
          T.thumb[i * 3] = (tbuf[i] << 3) & 0xff;
          T.thumb[i * 3 + 1] = (tbuf[i] >> 5 << 2) & 0xff;
          T.thumb[i * 3 + 2] = (tbuf[i] >> 11 << 3) & 0xff;
        }
        free(tbuf);
        T.tlength = T.tcolors * tlength;
        T.tformat = LIBRAW_THUMBNAIL_BITMAP;
        SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
        return 0;
      }
      else if (write_thumb == &LibRaw::ppm_thumb)
      {
        if (t_bytesps > 1)
          throw LIBRAW_EXCEPTION_IO_CORRUPT; // 8-bit thumb, but parsed for more
                                             // bits
        int t_length = T.twidth * T.theight * t_colors;

        if (T.tlength &&
            (int)T.tlength < t_length) // try to find tiff ifd with needed offset
        {
          int pifd = find_ifd_by_offset(libraw_internal_data.internal_data.toffset);
          if (pifd >= 0 && tiff_ifd[pifd].strip_offsets_count &&
              tiff_ifd[pifd].strip_byte_counts_count)
          {
            // We found it, calculate final size
            unsigned total_size = 0;
            for (int i = 0; i < tiff_ifd[pifd].strip_byte_counts_count; i++)
              total_size += tiff_ifd[pifd].strip_byte_counts[i];
            if (total_size != (unsigned)t_length) // recalculate colors
            {
              if (total_size == T.twidth * T.tlength * 3)
                T.tcolors = 3;
              else if (total_size == T.twidth * T.tlength)
                T.tcolors = 1;
            }
            T.tlength = total_size;
            if (T.thumb)
              free(T.thumb);
            T.thumb = (char *)malloc(T.tlength);
            merror(T.thumb, "ppm_thumb()");

            char *dest = T.thumb;
            INT64 pos = ID.input->tell();

            for (int i = 0; i < tiff_ifd[pifd].strip_byte_counts_count &&
                            i < tiff_ifd[pifd].strip_offsets_count;
                 i++)
            {
              int remain = T.tlength;
              int sz = tiff_ifd[pifd].strip_byte_counts[i];
              int off = tiff_ifd[pifd].strip_offsets[i];
              if (off >= 0 && off + sz <= ID.input->size() && sz <= remain)
              {
                ID.input->seek(off, SEEK_SET);
                ID.input->read(dest, sz, 1);
                remain -= sz;
                dest += sz;
              }
            }
            ID.input->seek(pos, SEEK_SET);
            T.tformat = LIBRAW_THUMBNAIL_BITMAP;
            SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
            return 0;
          }
        }

        if (!T.tlength)
          T.tlength = t_length;
        if (T.thumb)
          free(T.thumb);

        T.thumb = (char *)malloc(T.tlength);
        if (!T.tcolors)
          T.tcolors = t_colors;
        merror(T.thumb, "ppm_thumb()");

        ID.input->read(T.thumb, 1, T.tlength);

        T.tformat = LIBRAW_THUMBNAIL_BITMAP;
        SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
        return 0;
      }
      else if (write_thumb == &LibRaw::ppm16_thumb)
      {
        if (t_bytesps > 2)
          throw LIBRAW_EXCEPTION_IO_CORRUPT; // 16-bit thumb, but parsed for
                                             // more bits
        int o_bps = (imgdata.params.raw_processing_options &
                     LIBRAW_PROCESSING_USE_PPM16_THUMBS)
                        ? 2
                        : 1;
        int o_length = T.twidth * T.theight * t_colors * o_bps;
        int i_length = T.twidth * T.theight * t_colors * 2;
        if (!T.tlength)
          T.tlength = o_length;
        ushort *t_thumb = (ushort *)calloc(i_length, 1);
        ID.input->read(t_thumb, 1, i_length);
        if ((libraw_internal_data.unpacker_data.order == 0x4949) ==
            (ntohs(0x1234) == 0x1234))
          swab((char *)t_thumb, (char *)t_thumb, i_length);

        if (T.thumb)
          free(T.thumb);
        if ((imgdata.params.raw_processing_options &
             LIBRAW_PROCESSING_USE_PPM16_THUMBS))
        {
          T.thumb = (char *)t_thumb;
          T.tformat = LIBRAW_THUMBNAIL_BITMAP16;
        }
        else
        {
          T.thumb = (char *)malloc(o_length);
          merror(T.thumb, "ppm_thumb()");
          for (int i = 0; i < o_length; i++)
            T.thumb[i] = t_thumb[i] >> 8;
          free(t_thumb);
          T.tformat = LIBRAW_THUMBNAIL_BITMAP;
        }
        SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
        return 0;
      }
#ifdef USE_X3FTOOLS
	  else if (write_thumb == &LibRaw::x3f_thumb_loader)
      {
        x3f_thumb_loader();
        SET_PROC_FLAG(LIBRAW_PROGRESS_THUMB_LOAD);
        return 0;
      }
#endif
      else
      {
        return LIBRAW_UNSUPPORTED_THUMBNAIL;
      }
    }
    // last resort
    return LIBRAW_UNSUPPORTED_THUMBNAIL;
  }
  catch (LibRaw_exceptions err)
  {
    EXCEPTION_HANDLER(err);
  }
}