Commit message: More gmalloc → gmallocn
CVE: CVE-2009-3605
CWE: CWE-189
Vulnerability description: Multiple integer overflows in Poppler 0.10.5 and earlier allow remote attackers to cause a denial of service (application crash) or possibly execute arbitrary code via a crafted PDF file, related to (1) glib/poppler-page.cc; (2) ArthurOutputDev.cc, (3) CairoOutputDev.cc, (4) GfxState.cc, (5) JBIG2Stream.cc, (6) PSOutputDev.cc, and (7) SplashOutputDev.cc in poppler/; and (8) SplashBitmap.cc, (9) Splash.cc, and (10) SplashFTFont.cc in splash/.  NOTE: this may overlap CVE-2009-0791.

Project: poppler
File: Splash.cc

void Splash::clear(SplashColorPtr color, Guchar alpha) {
  SplashColorPtr row, p;
  Guchar mono;
  int x, y;

  switch (bitmap->mode) {
  case splashModeMono1:
    mono = (color[0] & 0x80) ? 0xff : 0x00;
    if (bitmap->rowSize < 0) {
      memset(bitmap->data + bitmap->rowSize * (bitmap->height - 1),
	     mono, -bitmap->rowSize * bitmap->height);
    } else {
      memset(bitmap->data, mono, bitmap->rowSize * bitmap->height);
    }
    break;
  case splashModeMono8:
    if (bitmap->rowSize < 0) {
      memset(bitmap->data + bitmap->rowSize * (bitmap->height - 1),
	     color[0], -bitmap->rowSize * bitmap->height);
    } else {
      memset(bitmap->data, color[0], bitmap->rowSize * bitmap->height);
    }
    break;
  case splashModeRGB8:
    if (color[0] == color[1] && color[1] == color[2]) {
      if (bitmap->rowSize < 0) {
	memset(bitmap->data + bitmap->rowSize * (bitmap->height - 1),
	       color[0], -bitmap->rowSize * bitmap->height);
      } else {
	memset(bitmap->data, color[0], bitmap->rowSize * bitmap->height);
      }
    } else {
      row = bitmap->data;
      for (y = 0; y < bitmap->height; ++y) {
	p = row;
	for (x = 0; x < bitmap->width; ++x) {
	  *p++ = color[2];
	  *p++ = color[1];
	  *p++ = color[0];
	}
	row += bitmap->rowSize;
      }
    }
    break;
  case splashModeXBGR8:
    if (color[0] == color[1] && color[1] == color[2]) {
      if (bitmap->rowSize < 0) {
	memset(bitmap->data + bitmap->rowSize * (bitmap->height - 1),
	       color[0], -bitmap->rowSize * bitmap->height);
      } else {
	memset(bitmap->data, color[0], bitmap->rowSize * bitmap->height);
      }
    } else {
      row = bitmap->data;
      for (y = 0; y < bitmap->height; ++y) {
	p = row;
	for (x = 0; x < bitmap->width; ++x) {
	  *p++ = color[0];
	  *p++ = color[1];
	  *p++ = color[2];
	  *p++ = 255;
	}
	row += bitmap->rowSize;
      }
    }
    break;
  case splashModeBGR8:
    if (color[0] == color[1] && color[1] == color[2]) {
      if (bitmap->rowSize < 0) {
	memset(bitmap->data + bitmap->rowSize * (bitmap->height - 1),
	       color[0], -bitmap->rowSize * bitmap->height);
      } else {
	memset(bitmap->data, color[0], bitmap->rowSize * bitmap->height);
      }
    } else {
      row = bitmap->data;
      for (y = 0; y < bitmap->height; ++y) {
	p = row;
	for (x = 0; x < bitmap->width; ++x) {
	  *p++ = color[0];
	  *p++ = color[1];
	  *p++ = color[2];
	}
	row += bitmap->rowSize;
      }
    }
    break;
#if SPLASH_CMYK
  case splashModeCMYK8:
    if (color[0] == color[1] && color[1] == color[2] && color[2] == color[3]) {
      if (bitmap->rowSize < 0) {
	memset(bitmap->data + bitmap->rowSize * (bitmap->height - 1),
	       color[0], -bitmap->rowSize * bitmap->height);
      } else {
	memset(bitmap->data, color[0], bitmap->rowSize * bitmap->height);
      }
    } else {
      row = bitmap->data;
      for (y = 0; y < bitmap->height; ++y) {
	p = row;
	for (x = 0; x < bitmap->width; ++x) {
	  *p++ = color[0];
	  *p++ = color[1];
	  *p++ = color[2];
	  *p++ = color[3];
	}
	row += bitmap->rowSize;
      }
    }
    break;
#endif
  }

  if (bitmap->alpha) {
    memset(bitmap->alpha, alpha, bitmap->width * bitmap->height);
  }

  updateModX(0);
  updateModY(0);
  updateModX(bitmap->width - 1);
  updateModY(bitmap->height - 1);
}

Project: poppler
File: Splash.cc

void Splash::compositeBackground(SplashColorPtr color) {
  SplashColorPtr p;
  Guchar *q;
  Guchar alpha, alpha1, c, color0, color1, color2;
#if SPLASH_CMYK
  Guchar color3;
#endif
  int x, y, mask;

  switch (bitmap->mode) {
  case splashModeMono1:
    color0 = color[0];
    for (y = 0; y < bitmap->height; ++y) {
      p = &bitmap->data[y * bitmap->rowSize];
      q = &bitmap->alpha[y * bitmap->width];
      mask = 0x80;
      for (x = 0; x < bitmap->width; ++x) {
	alpha = *q++;
	alpha1 = 255 - alpha;
	c = (*p & mask) ? 0xff : 0x00;
	c = div255(alpha1 * color0 + alpha * c);
	if (c & 0x80) {
	  *p |= mask;
	} else {
	  *p &= ~mask;
	}
	if (!(mask >>= 1)) {
	  mask = 0x80;
	  ++p;
	}
      }
    }
    break;
  case splashModeMono8:
    color0 = color[0];
    for (y = 0; y < bitmap->height; ++y) {
      p = &bitmap->data[y * bitmap->rowSize];
      q = &bitmap->alpha[y * bitmap->width];
      for (x = 0; x < bitmap->width; ++x) {
	alpha = *q++;
	alpha1 = 255 - alpha;
	p[0] = div255(alpha1 * color0 + alpha * p[0]);
	++p;
      }
    }
    break;
  case splashModeRGB8:
  case splashModeBGR8:
    color0 = color[0];
    color1 = color[1];
    color2 = color[2];
    for (y = 0; y < bitmap->height; ++y) {
      p = &bitmap->data[y * bitmap->rowSize];
      q = &bitmap->alpha[y * bitmap->width];
      for (x = 0; x < bitmap->width; ++x) {
	alpha = *q++;
	alpha1 = 255 - alpha;
	p[0] = div255(alpha1 * color0 + alpha * p[0]);
	p[1] = div255(alpha1 * color1 + alpha * p[1]);
	p[2] = div255(alpha1 * color2 + alpha * p[2]);
	p += 3;
      }
    }
    break;
  case splashModeXBGR8:
    color0 = color[0];
    color1 = color[1];
    color2 = color[2];
    for (y = 0; y < bitmap->height; ++y) {
      p = &bitmap->data[y * bitmap->rowSize];
      q = &bitmap->alpha[y * bitmap->width];
      for (x = 0; x < bitmap->width; ++x) {
	alpha = *q++;
	alpha1 = 255 - alpha;
	p[0] = div255(alpha1 * color0 + alpha * p[0]);
	p[1] = div255(alpha1 * color1 + alpha * p[1]);
	p[2] = div255(alpha1 * color2 + alpha * p[2]);
	p[3] = 255;
	p += 4;
      }
    }
    break;
#if SPLASH_CMYK
  case splashModeCMYK8:
    color0 = color[0];
    color1 = color[1];
    color2 = color[2];
    color3 = color[3];
    for (y = 0; y < bitmap->height; ++y) {
      p = &bitmap->data[y * bitmap->rowSize];
      q = &bitmap->alpha[y * bitmap->width];
      for (x = 0; x < bitmap->width; ++x) {
	alpha = *q++;
	alpha1 = 255 - alpha;
	p[0] = div255(alpha1 * color0 + alpha * p[0]);
	p[1] = div255(alpha1 * color1 + alpha * p[1]);
	p[2] = div255(alpha1 * color2 + alpha * p[2]);
	p[3] = div255(alpha1 * color3 + alpha * p[3]);
	p += 4;
      }
    }
    break;
#endif
  }
  memset(bitmap->alpha, 255, bitmap->width * bitmap->height);
}

Project: poppler
File: Splash.cc

SplashError Splash::fill(SplashPath *path, GBool eo) {
  if (debugMode) {
    printf("fill [eo:%d]:\n", eo);
    dumpPath(path);
  }
  return fillWithPattern(path, eo, state->fillPattern, state->fillAlpha);
}

Project: poppler
File: Splash.cc

SplashCoord Splash::getFillAlpha() {
  return state->fillAlpha;
}

Project: poppler
File: Splash.cc

int Splash::getLineJoin() {
  return state->lineJoin;
}

Project: poppler

inline void Splash::pipeRun(SplashPipe *pipe) {
  Guchar aSrc, aDest, alpha2, alpha0, aResult;
  SplashColor cDest, cBlend;
  Guchar cResult0, cResult1, cResult2, cResult3;



  if (pipe->pattern) {
    pipe->pattern->getColor(pipe->x, pipe->y, pipe->cSrcVal);
  }

  if (pipe->noTransparency && !state->blendFunc) {


    switch (bitmap->mode) {
    case splashModeMono1:
      cResult0 = pipe->cSrc[0];
      if (state->screen->test(pipe->x, pipe->y, cResult0)) {
	*pipe->destColorPtr |= pipe->destColorMask;
      } else {
	*pipe->destColorPtr &= ~pipe->destColorMask;
      }
      if (!(pipe->destColorMask >>= 1)) {
	pipe->destColorMask = 0x80;
	++pipe->destColorPtr;
      }
      break;
    case splashModeMono8:
      *pipe->destColorPtr++ = pipe->cSrc[0];
      break;
    case splashModeRGB8:
      *pipe->destColorPtr++ = pipe->cSrc[0];
      *pipe->destColorPtr++ = pipe->cSrc[1];
      *pipe->destColorPtr++ = pipe->cSrc[2];
      break;
    case splashModeXBGR8:
      *pipe->destColorPtr++ = pipe->cSrc[2];
      *pipe->destColorPtr++ = pipe->cSrc[1];
      *pipe->destColorPtr++ = pipe->cSrc[0];
      *pipe->destColorPtr++ = 255;
      break;
    case splashModeBGR8:
      *pipe->destColorPtr++ = pipe->cSrc[2];
      *pipe->destColorPtr++ = pipe->cSrc[1];
      *pipe->destColorPtr++ = pipe->cSrc[0];
      break;
#if SPLASH_CMYK
    case splashModeCMYK8:
      *pipe->destColorPtr++ = pipe->cSrc[0];
      *pipe->destColorPtr++ = pipe->cSrc[1];
      *pipe->destColorPtr++ = pipe->cSrc[2];
      *pipe->destColorPtr++ = pipe->cSrc[3];
      break;
#endif
    }
    if (pipe->destAlphaPtr) {
      *pipe->destAlphaPtr++ = 255;
    }

  } else {


    switch (bitmap->mode) {
    case splashModeMono1:
      cDest[0] = (*pipe->destColorPtr & pipe->destColorMask) ? 0xff : 0x00;
      break;
    case splashModeMono8:
      cDest[0] = *pipe->destColorPtr;
      break;
    case splashModeRGB8:
      cDest[0] = pipe->destColorPtr[0];
      cDest[1] = pipe->destColorPtr[1];
      cDest[2] = pipe->destColorPtr[2];
      break;
    case splashModeXBGR8:
      cDest[0] = pipe->destColorPtr[2];
      cDest[1] = pipe->destColorPtr[1];
      cDest[2] = pipe->destColorPtr[0];
      cDest[3] = 255;
      break;
    case splashModeBGR8:
      cDest[0] = pipe->destColorPtr[2];
      cDest[1] = pipe->destColorPtr[1];
      cDest[2] = pipe->destColorPtr[0];
      break;
#if SPLASH_CMYK
    case splashModeCMYK8:
      cDest[0] = pipe->destColorPtr[0];
      cDest[1] = pipe->destColorPtr[1];
      cDest[2] = pipe->destColorPtr[2];
      cDest[3] = pipe->destColorPtr[3];
      break;
#endif
    }
    if (pipe->destAlphaPtr) {
      aDest = *pipe->destAlphaPtr;
    } else {
      aDest = 0xff;
    }


    if (state->blendFunc) {
      (*state->blendFunc)(pipe->cSrc, cDest, cBlend, bitmap->mode);
    }


    if (state->softMask) {
      if (pipe->usesShape) {
	aSrc = (Guchar)splashRound(pipe->aInput * *pipe->softMaskPtr++
				   * pipe->shape);
      } else {
	aSrc = (Guchar)splashRound(pipe->aInput * *pipe->softMaskPtr++);
      }
    } else if (pipe->usesShape) {
      aSrc = (Guchar)splashRound(pipe->aInput * pipe->shape);
    } else {
      aSrc = pipe->aSrc;
    }


    if (pipe->noTransparency) {
      alpha2 = aResult = 255;
    } else {
      aResult = aSrc + aDest - div255(aSrc * aDest);

      if (pipe->alpha0Ptr) {
	alpha0 = *pipe->alpha0Ptr++;
	alpha2 = aResult + alpha0 - div255(aResult * alpha0);
      } else {
	alpha2 = aResult;
      }
    }


    cResult0 = cResult1 = cResult2 = cResult3 = 0; // make gcc happy

    switch (pipe->resultColorCtrl) {

#if SPLASH_CMYK
    case splashPipeResultColorNoAlphaBlendCMYK:
      cResult3 = div255((255 - aDest) * pipe->cSrc[3] + aDest * cBlend[3]);
#endif
    case splashPipeResultColorNoAlphaBlendRGB:
      cResult2 = div255((255 - aDest) * pipe->cSrc[2] + aDest * cBlend[2]);
      cResult1 = div255((255 - aDest) * pipe->cSrc[1] + aDest * cBlend[1]);
    case splashPipeResultColorNoAlphaBlendMono:
      cResult0 = div255((255 - aDest) * pipe->cSrc[0] + aDest * cBlend[0]);
      break;

    case splashPipeResultColorAlphaNoBlendMono:
      if (alpha2 == 0) {
	cResult0 = 0;
      } else {
	cResult0 = (Guchar)(((alpha2 - aSrc) * cDest[0] +
			     aSrc * pipe->cSrc[0]) / alpha2);
      }
      break;
    case splashPipeResultColorAlphaNoBlendRGB:
      if (alpha2 == 0) {
	cResult0 = 0;
	cResult1 = 0;
	cResult2 = 0;
      } else {
	cResult0 = (Guchar)(((alpha2 - aSrc) * cDest[0] +
			     aSrc * pipe->cSrc[0]) / alpha2);
	cResult1 = (Guchar)(((alpha2 - aSrc) * cDest[1] +
			     aSrc * pipe->cSrc[1]) / alpha2);
	cResult2 = (Guchar)(((alpha2 - aSrc) * cDest[2] +
			     aSrc * pipe->cSrc[2]) / alpha2);
      }
      break;
#if SPLASH_CMYK
    case splashPipeResultColorAlphaNoBlendCMYK:
      if (alpha2 == 0) {
	cResult0 = 0;
	cResult1 = 0;
	cResult2 = 0;
	cResult3 = 0;
      } else {
	cResult0 = (Guchar)(((alpha2 - aSrc) * cDest[0] +
			     aSrc * pipe->cSrc[0]) / alpha2);
	cResult1 = (Guchar)(((alpha2 - aSrc) * cDest[1] +
			     aSrc * pipe->cSrc[1]) / alpha2);
	cResult2 = (Guchar)(((alpha2 - aSrc) * cDest[2] +
			     aSrc * pipe->cSrc[2]) / alpha2);
	cResult3 = (Guchar)(((alpha2 - aSrc) * cDest[3] +
			     aSrc * pipe->cSrc[3]) / alpha2);
      }
      break;
#endif

    case splashPipeResultColorAlphaBlendMono:
      if (alpha2 == 0) {
	cResult0 = 0;
      } else {
	cResult0 = (Guchar)(((alpha2 - aSrc) * cDest[0] +
			     aSrc * ((255 - aDest) * pipe->cSrc[0] +
				     aDest * cBlend[0]) / 255) /
			    alpha2);
      }
      break;
    case splashPipeResultColorAlphaBlendRGB:
      if (alpha2 == 0) {
	cResult0 = 0;
	cResult1 = 0;
	cResult2 = 0;
      } else {
	cResult0 = (Guchar)(((alpha2 - aSrc) * cDest[0] +
			     aSrc * ((255 - aDest) * pipe->cSrc[0] +
				     aDest * cBlend[0]) / 255) /
			    alpha2);
	cResult1 = (Guchar)(((alpha2 - aSrc) * cDest[1] +
			     aSrc * ((255 - aDest) * pipe->cSrc[1] +
				     aDest * cBlend[1]) / 255) /
			    alpha2);
	cResult2 = (Guchar)(((alpha2 - aSrc) * cDest[2] +
			     aSrc * ((255 - aDest) * pipe->cSrc[2] +
				     aDest * cBlend[2]) / 255) /
			    alpha2);
      }
      break;
#if SPLASH_CMYK
    case splashPipeResultColorAlphaBlendCMYK:
      if (alpha2 == 0) {
	cResult0 = 0;
	cResult1 = 0;
	cResult2 = 0;
	cResult3 = 0;
      } else {
	cResult0 = (Guchar)(((alpha2 - aSrc) * cDest[0] +
			     aSrc * ((255 - aDest) * pipe->cSrc[0] +
				     aDest * cBlend[0]) / 255) /
			    alpha2);
	cResult1 = (Guchar)(((alpha2 - aSrc) * cDest[1] +
			     aSrc * ((255 - aDest) * pipe->cSrc[1] +
				     aDest * cBlend[1]) / 255) /
			    alpha2);
	cResult2 = (Guchar)(((alpha2 - aSrc) * cDest[2] +
			     aSrc * ((255 - aDest) * pipe->cSrc[2] +
				     aDest * cBlend[2]) / 255) /
			    alpha2);
	cResult3 = (Guchar)(((alpha2 - aSrc) * cDest[3] +
			     aSrc * ((255 - aDest) * pipe->cSrc[3] +
				     aDest * cBlend[3]) / 255) /
			    alpha2);
      }
      break;
#endif
    }


    if (aResult != 0) {
      switch (pipe->nonIsolatedGroup) {
#if SPLASH_CMYK
      case 4:
	cResult3 += (cResult3 - cDest[3]) * aDest *
	            (255 - aResult) / (255 * aResult);
#endif
      case 3:
	cResult2 += (cResult2 - cDest[2]) * aDest *
	            (255 - aResult) / (255 * aResult);
	cResult1 += (cResult1 - cDest[1]) * aDest *
	            (255 - aResult) / (255 * aResult);
      case 1:
	cResult0 += (cResult0 - cDest[0]) * aDest *
	            (255 - aResult) / (255 * aResult);
      case 0:
	break;
      }
    }


    switch (bitmap->mode) {
    case splashModeMono1:
      if (state->screen->test(pipe->x, pipe->y, cResult0)) {
	*pipe->destColorPtr |= pipe->destColorMask;
      } else {
	*pipe->destColorPtr &= ~pipe->destColorMask;
      }
      if (!(pipe->destColorMask >>= 1)) {
	pipe->destColorMask = 0x80;
	++pipe->destColorPtr;
      }
      break;
    case splashModeMono8:
      *pipe->destColorPtr++ = cResult0;
      break;
    case splashModeRGB8:
      *pipe->destColorPtr++ = cResult0;
      *pipe->destColorPtr++ = cResult1;
      *pipe->destColorPtr++ = cResult2;
      break;
    case splashModeXBGR8:
      *pipe->destColorPtr++ = cResult2;
      *pipe->destColorPtr++ = cResult1;
      *pipe->destColorPtr++ = cResult0;
      *pipe->destColorPtr++ = 255;
      break;
    case splashModeBGR8:
      *pipe->destColorPtr++ = cResult2;
      *pipe->destColorPtr++ = cResult1;
      *pipe->destColorPtr++ = cResult0;
      break;
#if SPLASH_CMYK
    case splashModeCMYK8:
      *pipe->destColorPtr++ = cResult0;
      *pipe->destColorPtr++ = cResult1;
      *pipe->destColorPtr++ = cResult2;
      *pipe->destColorPtr++ = cResult3;
      break;
#endif
    }
    if (pipe->destAlphaPtr) {
      *pipe->destAlphaPtr++ = aResult;
    }

  }

  ++pipe->x;
}

Project: poppler
File: Splash.cc

void Splash::setLineJoin(int lineJoin) {
  state->lineJoin = lineJoin;
}

Project: poppler

SplashError Splash::xorFill(SplashPath *path, GBool eo) {
  SplashPipe pipe;
  SplashXPath *xPath;
  SplashXPathScanner *scanner;
  int xMinI, yMinI, xMaxI, yMaxI, x0, x1, y;
  SplashClipResult clipRes, clipRes2;
  SplashBlendFunc origBlendFunc;

  if (path->length == 0) {
    return splashErrEmptyPath;
  }
  xPath = new SplashXPath(path, state->matrix, state->flatness, gTrue);
  xPath->sort();
  scanner = new SplashXPathScanner(xPath, eo);

  scanner->getBBox(&xMinI, &yMinI, &xMaxI, &yMaxI);

  if ((clipRes = state->clip->testRect(xMinI, yMinI, xMaxI, yMaxI))
      != splashClipAllOutside) {

    if (yMinI < state->clip->getYMinI()) {
      yMinI = state->clip->getYMinI();
    }
    if (yMaxI > state->clip->getYMaxI()) {
      yMaxI = state->clip->getYMaxI();
    }

    origBlendFunc = state->blendFunc;
    state->blendFunc = &blendXor;
    pipeInit(&pipe, 0, yMinI, state->fillPattern, NULL, 1, gFalse, gFalse);

    for (y = yMinI; y <= yMaxI; ++y) {
      while (scanner->getNextSpan(y, &x0, &x1)) {
	if (clipRes == splashClipAllInside) {
	  drawSpan(&pipe, x0, x1, y, gTrue);
	} else {
	  if (x0 < state->clip->getXMinI()) {
	    x0 = state->clip->getXMinI();
	  }
	  if (x1 > state->clip->getXMaxI()) {
	    x1 = state->clip->getXMaxI();
	  }
	  clipRes2 = state->clip->testSpan(x0, x1, y);
	  drawSpan(&pipe, x0, x1, y, clipRes2 == splashClipAllInside);
	}
      }
    }
    state->blendFunc = origBlendFunc;
  }
  opClipRes = clipRes;

  delete scanner;
  delete xPath;
  return splashOk;
}