Commit message: check for valid Huf code lengths (#849)

* check for valid Huf code lengths
* test non-fast huf decoder in testHuf

Signed-off-by: Peter Hillman <peterh@wetafx.co.nz>
CVE: CVE-2021-20304
CWE: CWE-190
Vulnerability description: A flaw was found in OpenEXR's hufDecode functionality. This flaw allows an attacker who can pass a crafted file to be processed by OpenEXR, to trigger an undefined right shift error. The highest threat from this vulnerability is to system availability.

Project: openexr
File: ImfHuf.cpp

countFrequencies (Int64 freq[HUF_ENCSIZE],
		  const unsigned short data[/*n*/],
		  int n)
#endif
{
    for (int i = 0; i < HUF_ENCSIZE; ++i)
	freq[i] = 0;

    for (int i = 0; i < n; ++i)
	++freq[data[i]];
}

Project: openexr
File: testHuf.cpp

compressUncompressSubset(const unsigned short raw[], int n)
{
    // Test various subsets of the data set

    Array <char> compressed (3 * n + 4 * 65536);
    Array <unsigned short> uncompressed (n);

    int maxOffset = 16;
    if (n <= maxOffset) {
        maxOffset = n-1;
    }

    for (int offset=1; offset<maxOffset; ++offset) {

        int maxLength = n - offset;
        int minLength = maxLength - 16;
        if (minLength < 1) minLength = 1;

        for (int length=minLength; length<=maxLength; ++length) {
            cout << "compressing " << flush;

            int nCompressed = hufCompress (raw + offset, length, compressed + offset);

            cout << "uncompressing " << flush;

            hufUncompress (compressed + offset, nCompressed, uncompressed + offset, length);

            for (int i = 0; i < length; ++i)
                assert (uncompressed[offset+i] == raw[offset+i]);

            cout << sizeof (raw[0]) * length << " bytes, compressed " << nCompressed  << " bytes ";
            cout << "(+" << offset << ",-" << n-offset-length << ")" << endl;
        }
    }
}

Project: openexr
File: testHuf.cpp

compressVerify (const unsigned short raw[], 
                int n,
                const unsigned int dekHash)
{
    Array <char> compressed (3 * n + 4 * 65536);

    int nCompressed = hufCompress (raw, n, compressed);
    
    //
    // This hash algorithm proposed by Donald E. Knuth in 
    // The Art Of Computer Programming Volume 3,
    // under the topic of sorting and search chapter 6.4. 
    //
    unsigned int compressedHash = nCompressed;
    const unsigned char* cptr = reinterpret_cast<const unsigned char*>( (const char*) compressed);
    for (int i = 0; i < nCompressed; ++i)
    {
        compressedHash = 
            ((compressedHash << 5) ^ (compressedHash >> 27)) ^ (*cptr++);
    }

    cout << "verifying compressed checksum hash = " 
        << compressedHash << std::endl;

    if (compressedHash != dekHash)
    {
       cout << "hash verification failed. Got " << compressedHash << " expected " << dekHash << std::endl;
       const unsigned char* cptr = reinterpret_cast<const unsigned char*>( (const char*) compressed);
       for(int i = 0 ; i < nCompressed ; ++i )
       {
           cout << std::hex << (0xFF & (int) (*cptr++));
           if ( (i & 0xF) ==0 )
           {
              cout << '\n';
           }
           else
           {
              cout << ' ';
           }
       }
       cout << "\n";
    }

    assert (compressedHash == dekHash);
}

Project: openexr
File: testHuf.cpp

fill1 (unsigned short data[/*n*/], int n, float bias, IMATH_NAMESPACE::Rand48 & rand48)
{
    for (int i = 0; i < n; ++i)
	data[i] = (unsigned short)
		  (pow (rand48.nextf(), double(bias)) * (USHRT_MAX + 1));
}

Project: openexr
File: testHuf.cpp

fill3 (unsigned short data[/*n*/], int n, int m)
{
    for (int i = 0; i < n; ++i)
	data[i] = m;
}

Project: openexr
File: ImfHuf.cpp

hufUncompress (const char compressed[],
	       int nCompressed,
	       unsigned short raw[],
	       int nRaw)
{
    //
    // need at least 20 bytes for header
    //
    if (nCompressed < 20 )
    {
	if (nRaw != 0)
	    notEnoughData();

	return;
    }

    int im = readUInt (compressed);
    int iM = readUInt (compressed + 4);
    // int tableLength = readUInt (compressed + 8);
    int nBits = readUInt (compressed + 12);

    if (im < 0 || im >= HUF_ENCSIZE || iM < 0 || iM >= HUF_ENCSIZE)
	invalidTableSize();

    const char *ptr = compressed + 20;

    uint64_t nBytes = (static_cast<uint64_t>(nBits)+7) / 8 ;

    if ( ptr + nBytes > compressed+nCompressed)
    {
        notEnoughData();
        return;
    }

    // 
    // Fast decoder needs at least 2x64-bits of compressed data, and
    // needs to be run-able on this platform. Otherwise, fall back
    // to the original decoder
    //

    if (FastHufDecoder::enabled() && nBits > 128)
    {
        FastHufDecoder fhd (ptr, nCompressed - (ptr - compressed), im, iM, iM);
        fhd.decode ((unsigned char*)ptr, nBits, raw, nRaw);
    }
    else
    {
        AutoArray <Int64, HUF_ENCSIZE> freq;
        AutoArray <HufDec, HUF_DECSIZE> hdec;

        hufClearDecTable (hdec);

        hufUnpackEncTable (&ptr,
                           nCompressed - (ptr - compressed),
                           im,
                           iM,
                           freq);

        try
        {
            if (nBits > 8 * (nCompressed - (ptr - compressed)))
                invalidNBits();

            hufBuildDecTable (freq, im, iM, hdec);
            hufDecode (freq, hdec, ptr, nBits, iM, nRaw, raw);
        }
        catch (...)
        {
            hufFreeDecTable (hdec);
            throw;
        }

        hufFreeDecTable (hdec);
    }
}

Project: openexr
File: testHuf.cpp

compressUncompress (const unsigned short raw[], int n)
{
    Array <char> compressed (3 * n + 4 * 65536);
    Array <unsigned short> uncompressed (n);

    cout << "compressing " << flush;

    int nCompressed = hufCompress (raw, n, compressed);

    cout << "uncompressing " << flush;

    hufUncompress (compressed, nCompressed, uncompressed, n);

    cout << "comparing: " << flush;

    for (int i = 0; i < n; ++i)
	assert (uncompressed[i] == raw[i]);

    cout << sizeof (raw[0]) * n << " bytes, compressed " <<
	    nCompressed  << " bytes" << endl;
}

Project: openexr
File: ImfHuf.cpp

hufDecode
    (const Int64 * 	hcode,	// i : encoding table
     const HufDec * 	hdecod,	// i : decoding table
     const char* 	in,	// i : compressed input buffer
     int		ni,	// i : input size (in bits)
     int		rlc,	// i : run-length code
     int		no,	// i : expected output size (in bytes)
     unsigned short*	out)	//  o: uncompressed output buffer
{
    Int64 c = 0;
    int lc = 0;
    unsigned short * outb = out;
    unsigned short * oe = out + no;
    const char * ie = in + (ni + 7) / 8; // input byte size

    //
    // Loop on input bytes
    //

    while (in < ie)
    {
	getChar (c, lc, in);

	//
	// Access decoding table
	//

	while (lc >= HUF_DECBITS)
	{
	    const HufDec pl = hdecod[(c >> (lc-HUF_DECBITS)) & HUF_DECMASK];

	    if (pl.len)
	    {
		//
		// Get short code
		//

		lc -= pl.len;

		if ( lc < 0 )
		{
			invalidCode(); // code length too long
		}
		getCode (pl.lit, rlc, c, lc, in, out, outb, oe);
	    }
	    else
	    {
		if (!pl.p)
		    invalidCode(); // wrong code

		//
		// Search long code
		//

		int j;

		for (j = 0; j < pl.lit; j++)
		{
		    int	l = hufLength (hcode[pl.p[j]]);

		    while (lc < l && in < ie)	// get more bits
			getChar (c, lc, in);

		    if (lc >= l)
		    {
			if (hufCode (hcode[pl.p[j]]) ==
				((c >> (lc - l)) & ((Int64(1) << l) - 1)))
			{
			    //
			    // Found : get long code
			    //

			    lc -= l;
			    getCode (pl.p[j], rlc, c, lc, in, out, outb, oe);
			    break;
			}
		    }
		}

		if (j == pl.lit)
		    invalidCode(); // Not found
	    }
	}
    }

    //
    // Get remaining (short) codes
    //

    int i = (8 - ni) & 7;
    c >>= i;
    lc -= i;

    while (lc > 0)
    {
	const HufDec pl = hdecod[(c << (HUF_DECBITS - lc)) & HUF_DECMASK];

	if (pl.len)
	{
	    lc -= pl.len;
            if ( lc < 0 )
            {
   	        invalidCode(); // code length too long
            }
	    getCode (pl.lit, rlc, c, lc, in, out, outb, oe);
	}
	else
	{
	    invalidCode(); // wrong (long) code
	}
    }

    if (out - outb != no)
	notEnoughData ();
}

Project: openexr
File: ImfHuf.cpp

hufCanonicalCodeTable (Int64 hcode[HUF_ENCSIZE])
#endif
{
    Int64 n[59];

    //
    // For each i from 0 through 58, count the
    // number of different codes of length i, and
    // store the count in n[i].
    //

    for (int i = 0; i <= 58; ++i)
	n[i] = 0;

    for (int i = 0; i < HUF_ENCSIZE; ++i)
	n[hcode[i]] += 1;

    //
    // For each i from 58 through 1, compute the
    // numerically lowest code with length i, and
    // store that code in n[i].
    //

    Int64 c = 0;

    for (int i = 58; i > 0; --i)
    {
	Int64 nc = ((c + n[i]) >> 1);
	n[i] = c;
	c = nc;
    }

    //
    // hcode[i] contains the length, l, of the
    // code for symbol i.  Assign the next available
    // code of length l to the symbol and store both
    // l and the code in hcode[i].
    //

    for (int i = 0; i < HUF_ENCSIZE; ++i)
    {
	int l = hcode[i];

	if (l > 0)
	    hcode[i] = l | (n[l]++ << 6);
    }
}

Project: openexr
File: testHuf.cpp

fill5 (unsigned short data[/*n*/], int n)
{
    for (int i = 0; i < n; ++i)
	data[i] = 0;

    int j = 0, k = 0;

    for (int i = 0; i < n; ++i)
    {
	data[i] = j;
	j = j + k;
	k = k + 1;

	if (j > USHRT_MAX)
	    break;
    }
}

Project: openexr
File: testHuf.cpp

fill4 (unsigned short data[/*n*/], int n)
{
    for (int i = 0; i < n; ++i)
	data[i] = i & USHRT_MAX;
}

Project: openexr
File: testHuf.cpp

testHuf (const std::string&)
{
    try
    {
	cout << "Testing Huffman encoder" << endl;

	IMATH_NAMESPACE::Rand48 rand48 (0);

        //
        // FastHufDecoder is used for more than 128 bits, so first test with fewer than 128 bits,
        // then test FastHufDecoder
        //
        for (int pass = 0 ; pass < 2 ; ++pass)
        {

            int N = pass==0 ? 12 : 1000000;
            Array <unsigned short> raw (N);

            fill1 (raw, N, 1, rand48);	  // test various symbol distributions
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill1 (raw, N, 10, rand48);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill1 (raw, N, 100, rand48);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill1 (raw, N, 1000, rand48);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);

            fill2 (raw, N, 1, rand48);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill2 (raw, N, 10, rand48);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill2 (raw, N, 100, rand48);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill2 (raw, N, 1000, rand48);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);

            fill3 (raw, N, 0);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill3 (raw, N, 1);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill3 (raw, N, USHRT_MAX - 1);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);
            fill3 (raw, N, USHRT_MAX);
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);

            if (pass==1)
            {
                fill4 (raw, USHRT_MAX + 1);
                compressVerify(raw, USHRT_MAX + 1, HUF_COMPRESS_DEK_HASH_FOR_FILL4_USHRT_MAX_PLUS_ONE);

                compressUncompress (raw, USHRT_MAX + 1);
                compressUncompressSubset (raw, USHRT_MAX + 1);
                fill4 (raw, N);
                compressVerify(raw, N, HUF_COMPRESS_DEK_HASH_FOR_FILL4_N);
            }
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);

            fill4 (raw, 0);
            compressUncompress (raw, 0);	// test small input data sets
            fill4 (raw, 1);
            compressUncompress (raw, 1);
            fill4 (raw, 2);
            compressUncompress (raw, 2);
            fill4 (raw, 3);
            compressUncompress (raw, 3);

            fill5 (raw, N);			// test run-length coding of code table
            if (pass==1)
            {
                compressVerify(raw, N, HUF_COMPRESS_DEK_HASH_FOR_FILL5_N);
            }
            compressUncompress (raw, N);
            compressUncompressSubset (raw, N);

        }

	cout << "ok\n" << endl;
    }
    catch (const std::exception &e)
    {
	cerr << "ERROR -- caught exception: " << e.what() << endl;
	assert (false);
    }
}

Project: openexr
File: testHuf.cpp

fill2 (unsigned short data[/*n*/], int n, int m, IMATH_NAMESPACE::Rand48 & rand48)
{
    for (int i = 0; i < n; ++i)
	data[i] = 0;

    for (int i = 0; i < m; ++i)
	data[rand48.nexti() % n] = (unsigned short) (rand48.nextf() * (USHRT_MAX + 1));
}

Project: openexr
File: ImfHuf.cpp

    bool operator () (Int64 *a, Int64 *b)
    {
        return ((*a > *b) || ((*a == *b) && (a > b)));
    }