Commit message: Avoid overflow in calculateNumTiles when size=MAX_INT (#825)

* Avoid overflow in calculateNumTiles when size=MAX_INT

Signed-off-by: Cary Phillips <cary@ilm.com>

* Compute level size with 64 bits to avoid overflow

Signed-off-by: Cary Phillips <cary@ilm.com>
CVE: CVE-2021-3475
CWE: CWE-190
Vulnerability description: There is a flaw in OpenEXR in versions before 3.0.0-beta. An attacker who can submit a crafted file to be processed by OpenEXR could cause an integer overflow, potentially leading to problems with application availability.

Project: openexr
File: ImfTiledMisc.cpp

calculateNumYLevels (const TileDescription& tileDesc,
		     int minX, int maxX,
		     int minY, int maxY)
{
    int num = 0;

    switch (tileDesc.mode)
    {
      case ONE_LEVEL:

	num = 1;
	break;

      case MIPMAP_LEVELS:

	{
	  int w = maxX - minX + 1;
	  int h = maxY - minY + 1;
	  num = roundLog2 (std::max (w, h), tileDesc.roundingMode) + 1;
	}
        break;

      case RIPMAP_LEVELS:

	{
	  int h = maxY - minY + 1;
	  num = roundLog2 (h, tileDesc.roundingMode) + 1;
	}
	break;

      default:

	throw IEX_NAMESPACE::ArgExc ("Unknown LevelMode format.");
    }

    return num;
}

Project: openexr
File: ImfTiledMisc.cpp

precalculateTileInfo (const TileDescription& tileDesc,
		      int minX, int maxX,
		      int minY, int maxY,
		      int *&numXTiles, int *&numYTiles,
		      int &numXLevels, int &numYLevels)
{
    numXLevels = calculateNumXLevels(tileDesc, minX, maxX, minY, maxY);
    numYLevels = calculateNumYLevels(tileDesc, minX, maxX, minY, maxY);
    
    numXTiles = new int[numXLevels];
    numYTiles = new int[numYLevels];

    calculateNumTiles (numXTiles,
		       numXLevels,
		       minX, maxX,
		       tileDesc.xSize,
		       tileDesc.roundingMode);

    calculateNumTiles (numYTiles,
		       numYLevels,
		       minY, maxY,
		       tileDesc.ySize,
		       tileDesc.roundingMode);
}

Project: openexr
File: ImfTiledMisc.cpp

calculateBytesPerPixel (const Header &header)
{
    const ChannelList &channels = header.channels();

    size_t bytesPerPixel = 0;

    for (ChannelList::ConstIterator c = channels.begin();
	 c != channels.end();
	 ++c)
    {
	bytesPerPixel += pixelTypeSize (c.channel().type);
    }

    return bytesPerPixel;
}

Project: openexr
File: ImfTiledMisc.cpp

getTiledChunkOffsetTableSize(const Header& header)
{
    //
    // Save the dataWindow information
    //

    const Box2i &dataWindow = header.dataWindow();
    
    //
    // Precompute level and tile information.
    //

    int* numXTiles=nullptr;
    int* numYTiles=nullptr;
    int numXLevels;
    int numYLevels;
    try
    {
        precalculateTileInfo (header.tileDescription(),
                            dataWindow.min.x, dataWindow.max.x,
                            dataWindow.min.y, dataWindow.max.y,
                            numXTiles, numYTiles,
                            numXLevels, numYLevels);

        //
        // Calculate lineOffsetSize.
        //
        Int64 lineOffsetSize = 0;
        const TileDescription &desc = header.tileDescription();
        switch (desc.mode)
        {
            case ONE_LEVEL:
            case MIPMAP_LEVELS:
                for (int i = 0; i < numXLevels; i++)
                {
                    lineOffsetSize += static_cast<Int64>(numXTiles[i]) * static_cast<Int64>(numYTiles[i]);
                    if ( lineOffsetSize > static_cast<Int64>(std::numeric_limits<int>::max()) )
                    {
                        throw IEX_NAMESPACE::LogicExc("Maximum number of tiles exceeded");
                    }
                }
            break;
            case RIPMAP_LEVELS:
                for (int i = 0; i < numXLevels; i++)
                {
                    for (int j = 0; j < numYLevels; j++)
                    {
                        lineOffsetSize += static_cast<Int64>(numXTiles[i]) * static_cast<Int64>(numYTiles[j]);
                        if ( lineOffsetSize > static_cast<Int64>(std::numeric_limits<int>::max()) )
                        {
                            throw IEX_NAMESPACE::LogicExc("Maximum number of tiles exceeded");
                        }
                    }
                }
            break;
            case NUM_LEVELMODES :
                throw IEX_NAMESPACE::LogicExc("Bad level mode getting chunk offset table size");
        }
        delete[] numXTiles;
        delete[] numYTiles;

        return static_cast<int>(lineOffsetSize);

    }
    catch(...)
    {
        delete[] numXTiles;
        delete[] numYTiles;

        throw;
    }

}

Project: openexr
File: ImfTiledMisc.cpp

calculateBytesPerLine (const Header &header,
                       char* sampleCountBase,
                       int sampleCountXStride,
                       int sampleCountYStride,
                       int minX, int maxX,
                       int minY, int maxY,
                       std::vector<int>& xOffsets,
                       std::vector<int>& yOffsets,
                       std::vector<Int64>& bytesPerLine)
{
    const ChannelList &channels = header.channels();

    int pos = 0;
    for (ChannelList::ConstIterator c = channels.begin();
         c != channels.end();
         ++c, ++pos)
    {
        int xOffset = xOffsets[pos];
        int yOffset = yOffsets[pos];
        int i = 0;
        for (int y = minY - yOffset; y <= maxY - yOffset; y++, i++)
            for (int x = minX - xOffset; x <= maxX - xOffset; x++)
            {
                bytesPerLine[i] += sampleCount(sampleCountBase,
                                               sampleCountXStride,
                                               sampleCountYStride,
                                               x, y)
                                   * pixelTypeSize (c.channel().type);
            }
    }
}

Project: openexr
File: ImfTiledMisc.cpp

dataWindowForTile (const TileDescription &tileDesc,
		   int minX, int maxX,
		   int minY, int maxY,
		   int dx, int dy,
		   int lx, int ly)
{
    V2i tileMin = V2i (minX + dx * tileDesc.xSize,
		       minY + dy * tileDesc.ySize);

    V2i tileMax = tileMin + V2i (tileDesc.xSize - 1, tileDesc.ySize - 1);

    V2i levelMax = dataWindowForLevel
		       (tileDesc, minX, maxX, minY, maxY, lx, ly).max;

    tileMax = V2i (std::min (tileMax[0], levelMax[0]),
		   std::min (tileMax[1], levelMax[1]));

    return Box2i (tileMin, tileMax);
}

Project: openexr
File: ImfTiledMisc.cpp

floorLog2 (int x)
{
    //
    // For x > 0, floorLog2(y) returns floor(log(x)/log(2)).
    //

    int y = 0;

    while (x > 1)
    {
	y +=  1;
	x >>= 1;
    }

    return y;
}

Project: openexr
File: ImfTiledMisc.cpp

levelSize (int min, int max, int l, LevelRoundingMode rmode)
{
    if (l < 0)
	throw IEX_NAMESPACE::ArgExc ("Argument not in valid range.");

    int a = max - min + 1;
    int b = (1 << l);
    int size = a / b;

    if (rmode == ROUND_UP && size * b < a)
	size += 1;

    return std::max (size, 1);
}

Project: openexr
File: ImfTiledMisc.cpp

dataWindowForLevel (const TileDescription &tileDesc,
		    int minX, int maxX,
		    int minY, int maxY,
		    int lx, int ly)
{
    V2i levelMin = V2i (minX, minY);

    V2i levelMax = levelMin +
		   V2i (levelSize (minX, maxX, lx, tileDesc.roundingMode) - 1,
			levelSize (minY, maxY, ly, tileDesc.roundingMode) - 1);

    return Box2i(levelMin, levelMax);
}

Project: openexr
File: ImfTiledMisc.cpp

calculateNumTiles (int *numTiles,
		   int numLevels,
		   int min, int max,
		   int size,
		   LevelRoundingMode rmode)
{
    for (int i = 0; i < numLevels; i++)
    {
        // use 64 bits to avoid int overflow if size is large.
        Int64 l = levelSize (min, max, i, rmode);
        numTiles[i] = (l + size - 1) / size;
    }
}

Project: openexr
File: ImfTiledMisc.cpp

ceilLog2 (int x)
{
    //
    // For x > 0, ceilLog2(y) returns ceil(log(x)/log(2)).
    //

    int y = 0;
    int r = 0;

    while (x > 1)
    {
	if (x & 1)
	    r = 1;

	y +=  1;
	x >>= 1;
    }

    return y + r;
}

Project: openexr
File: ImfTiledMisc.cpp

calculateNumXLevels (const TileDescription& tileDesc,
		     int minX, int maxX,
		     int minY, int maxY)
{
    int num = 0;

    switch (tileDesc.mode)
    {
      case ONE_LEVEL:

	num = 1;
	break;

      case MIPMAP_LEVELS:

	{
	  int w = maxX - minX + 1;
	  int h = maxY - minY + 1;
	  num = roundLog2 (std::max (w, h), tileDesc.roundingMode) + 1;
	}
        break;

      case RIPMAP_LEVELS:

	{
	  int w = maxX - minX + 1;
	  num = roundLog2 (w, tileDesc.roundingMode) + 1;
	}
	break;

      default:

	throw IEX_NAMESPACE::ArgExc ("Unknown LevelMode format.");
    }

    return num;
}

Project: openexr
File: ImfTiledMisc.cpp

roundLog2 (int x, LevelRoundingMode rmode)
{
    return (rmode == ROUND_DOWN)? floorLog2 (x): ceilLog2 (x);
}