Commit message: src/common.c: Fix heap buffer overflows when writing strings in binheader

Fixes the following problems:
 1. Case 's' only enlarges the buffer by 16 bytes instead of size bytes.
 2. psf_binheader_writef() enlarges the header buffer (if needed) prior to the
    big switch statement by an amount (16 bytes) which is enough for all cases
    where only a single value gets added. Cases 's', 'S', 'p' however
    additionally write an arbitrary length block of data and again enlarge the
    buffer to the required amount. However, the required space calculation does
    not take into account the size of the length field which gets output before
    the data.
 3. Buffer size requirement calculation in case 'S' does not account for the
    padding byte ("size += (size & 1) ;" happens after the calculation which
    uses "size").
 4. Case 'S' can overrun the header buffer by 1 byte when no padding is
    involved
    ("memcpy (&(psf->header.ptr [psf->header.indx]), strptr, size + 1) ;" while
    the buffer is only guaranteed to have "size" space available).
 5. "psf->header.ptr [psf->header.indx] = 0 ;" in case 'S' always writes 1 byte
    beyond the space which is guaranteed to be allocated in the header buffer.
 6. Case 's' can overrun the provided source string by 1 byte if padding is
    involved ("memcpy (&(psf->header.ptr [psf->header.indx]), strptr, size) ;"
    where "size" is "strlen (strptr) + 1" (which includes the 0 terminator,
    plus optionally another 1 which is padding and not guaranteed to be
    readable via the source string pointer).

Closes: https://github.com/erikd/libsndfile/issues/292
CVE: CVE-2017-12562
CWE: CWE-119
Vulnerability description: Heap-based Buffer Overflow in the psf_binheader_writef function in common.c in libsndfile through 1.0.28 allows remote attackers to cause a denial of service (application crash) or possibly have unspecified other impact.

Project: libsndfile
File: common.c

psf_allocate (void)
{	SF_PRIVATE * psf ;

	if ((psf = calloc (1, sizeof (SF_PRIVATE))) == NULL)
		return	NULL ;

	if ((psf->header.ptr = calloc (1, INITAL_HEADER_SIZE)) == NULL)
	{	free (psf) ;
		return	NULL ;
		} ;
	psf->header.len = INITAL_HEADER_SIZE ;

	return psf ;
} /* psf_allocate */

Project: libsndfile
File: common.c

psf_bump_header_allocation (SF_PRIVATE * psf, sf_count_t needed)
{
	sf_count_t newlen, smallest = INITAL_HEADER_SIZE ;
	void * ptr ;

	newlen = (needed > psf->header.len) ? 2 * SF_MAX (needed, smallest) : 2 * psf->header.len ;

	if (newlen > 100 * 1024)
	{	psf_log_printf (psf, "Request for header allocation of %D denied.\n", newlen) ;
		return 1 ;
		}

	if ((ptr = realloc (psf->header.ptr, newlen)) == NULL)
	{	psf_log_printf (psf, "realloc (%p, %D) failed\n", psf->header.ptr, newlen) ;
		psf->error = SFE_MALLOC_FAILED ;
		return 1 ;
		} ;

	/* Always zero-out new header memory to avoid un-initializer memory accesses. */
	if (newlen > psf->header.len)
		memset ((char *) ptr + psf->header.len, 0, newlen - psf->header.len) ;

	psf->header.ptr = ptr ;
	psf->header.len = newlen ;
	return 0 ;
} /* psf_bump_header_allocation */

Project: libsndfile
File: common.c

psf_d2i_clip_array (const double *src, int *dest, int count, int normalize)
{	double			normfact, scaled_value ;

	normfact = normalize ? (8.0 * 0x10000000) : 1.0 ;

	while (--count >= 0)
	{	scaled_value = src [count] * normfact ;
		if (CPU_CLIPS_POSITIVE == 0 && scaled_value >= (1.0 * 0x7FFFFFFF))
		{	dest [count] = 0x7FFFFFFF ;
			continue ;
			} ;
		if (CPU_CLIPS_NEGATIVE == 0 && scaled_value <= (-8.0 * 0x10000000))
		{	dest [count] = 0x80000000 ;
			continue ;
			} ;

		dest [count] = lrint (scaled_value) ;
		} ;

	return ;
} /* psf_d2i_clip_array */

Project: libsndfile
File: common.c

psf_binheader_writef (SF_PRIVATE *psf, const char *format, ...)
{	va_list	argptr ;
	sf_count_t 		countdata ;
	unsigned long 	longdata ;
	unsigned int 	data ;
	float			floatdata ;
	double			doubledata ;
	void			*bindata ;
	size_t			size ;
	char			c, *strptr ;
	int				count = 0, trunc_8to4 ;

	trunc_8to4 = SF_FALSE ;

	va_start (argptr, format) ;

	while ((c = *format++))
	{
		if (psf->header.indx + 16 >= psf->header.len && psf_bump_header_allocation (psf, 16))
			return count ;

		switch (c)
		{	case ' ' : /* Do nothing. Just used to space out format string. */
					break ;

			case 'e' : /* All conversions are now from LE to host. */
					psf->rwf_endian = SF_ENDIAN_LITTLE ;
					break ;

			case 'E' : /* All conversions are now from BE to host. */
					psf->rwf_endian = SF_ENDIAN_BIG ;
					break ;

			case 't' : /* All 8 byte values now get written as 4 bytes. */
					trunc_8to4 = SF_TRUE ;
					break ;

			case 'T' : /* All 8 byte values now get written as 8 bytes. */
					trunc_8to4 = SF_FALSE ;
					break ;

			case 'm' :
					data = va_arg (argptr, unsigned int) ;
					header_put_marker (psf, data) ;
					count += 4 ;
					break ;

			case '1' :
					data = va_arg (argptr, unsigned int) ;
					header_put_byte (psf, data) ;
					count += 1 ;
					break ;

			case '2' :
					data = va_arg (argptr, unsigned int) ;
					if (psf->rwf_endian == SF_ENDIAN_BIG)
					{	header_put_be_short (psf, data) ;
						}
					else
					{	header_put_le_short (psf, data) ;
						} ;
					count += 2 ;
					break ;

			case '3' : /* tribyte */
					data = va_arg (argptr, unsigned int) ;
					if (psf->rwf_endian == SF_ENDIAN_BIG)
					{	header_put_be_3byte (psf, data) ;
						}
					else
					{	header_put_le_3byte (psf, data) ;
						} ;
					count += 3 ;
					break ;

			case '4' :
					data = va_arg (argptr, unsigned int) ;
					if (psf->rwf_endian == SF_ENDIAN_BIG)
					{	header_put_be_int (psf, data) ;
						}
					else
					{	header_put_le_int (psf, data) ;
						} ;
					count += 4 ;
					break ;

			case '8' :
					countdata = va_arg (argptr, sf_count_t) ;
					if (psf->rwf_endian == SF_ENDIAN_BIG && trunc_8to4 == SF_FALSE)
					{	header_put_be_8byte (psf, countdata) ;
						count += 8 ;
						}
					else if (psf->rwf_endian == SF_ENDIAN_LITTLE && trunc_8to4 == SF_FALSE)
					{	header_put_le_8byte (psf, countdata) ;
						count += 8 ;
						}
					else if (psf->rwf_endian == SF_ENDIAN_BIG && trunc_8to4 == SF_TRUE)
					{	longdata = countdata & 0xFFFFFFFF ;
						header_put_be_int (psf, longdata) ;
						count += 4 ;
						}
					else if (psf->rwf_endian == SF_ENDIAN_LITTLE && trunc_8to4 == SF_TRUE)
					{	longdata = countdata & 0xFFFFFFFF ;
						header_put_le_int (psf, longdata) ;
						count += 4 ;
						}
					break ;

			case 'f' :
					/* Floats are passed as doubles. Is this always true? */
					floatdata = (float) va_arg (argptr, double) ;
					if (psf->rwf_endian == SF_ENDIAN_BIG)
						float32_be_write (floatdata, psf->header.ptr + psf->header.indx) ;
					else
						float32_le_write (floatdata, psf->header.ptr + psf->header.indx) ;
					psf->header.indx += 4 ;
					count += 4 ;
					break ;

			case 'd' :
					doubledata = va_arg (argptr, double) ;
					if (psf->rwf_endian == SF_ENDIAN_BIG)
						double64_be_write (doubledata, psf->header.ptr + psf->header.indx) ;
					else
						double64_le_write (doubledata, psf->header.ptr + psf->header.indx) ;
					psf->header.indx += 8 ;
					count += 8 ;
					break ;

			case 's' :
					/* Write a C string (guaranteed to have a zero terminator). */
					strptr = va_arg (argptr, char *) ;
					size = strlen (strptr) + 1 ;

					if (psf->header.indx + 4 + (sf_count_t) size + (sf_count_t) (size & 1) > psf->header.len && psf_bump_header_allocation (psf, 4 + size + (size & 1)))
						return count ;

					if (psf->rwf_endian == SF_ENDIAN_BIG)
						header_put_be_int (psf, size + (size & 1)) ;
					else
						header_put_le_int (psf, size + (size & 1)) ;
					memcpy (&(psf->header.ptr [psf->header.indx]), strptr, size) ;
					size += (size & 1) ;
					psf->header.indx += size ;
					psf->header.ptr [psf->header.indx - 1] = 0 ;
					count += 4 + size ;
					break ;

			case 'S' :
					/*
					**	Write an AIFF style string (no zero terminator but possibly
					**	an extra pad byte if the string length is odd).
					*/
					strptr = va_arg (argptr, char *) ;
					size = strlen (strptr) ;
					if (psf->header.indx + 4 + (sf_count_t) size + (sf_count_t) (size & 1) > psf->header.len && psf_bump_header_allocation (psf, 4 + size + (size & 1)))
						return count ;
					if (psf->rwf_endian == SF_ENDIAN_BIG)
						header_put_be_int (psf, size) ;
					else
						header_put_le_int (psf, size) ;
					memcpy (&(psf->header.ptr [psf->header.indx]), strptr, size + (size & 1)) ;
					size += (size & 1) ;
					psf->header.indx += size ;
					count += 4 + size ;
					break ;

			case 'p' :
					/* Write a PASCAL string (as used by AIFF files).
					*/
					strptr = va_arg (argptr, char *) ;
					size = strlen (strptr) ;
					size = (size & 1) ? size : size + 1 ;
					size = (size > 254) ? 254 : size ;

					if (psf->header.indx + 1 + (sf_count_t) size > psf->header.len && psf_bump_header_allocation (psf, 1 + size))
						return count ;

					header_put_byte (psf, size) ;
					memcpy (&(psf->header.ptr [psf->header.indx]), strptr, size) ;
					psf->header.indx += size ;
					count += 1 + size ;
					break ;

			case 'b' :
					bindata	= va_arg (argptr, void *) ;
					size	= va_arg (argptr, size_t) ;

					if (psf->header.indx + (sf_count_t) size > psf->header.len && psf_bump_header_allocation (psf, size))
						return count ;

					memcpy (&(psf->header.ptr [psf->header.indx]), bindata, size) ;
					psf->header.indx += size ;
					count += size ;
					break ;

			case 'z' :
					size = va_arg (argptr, size_t) ;

					if (psf->header.indx + (sf_count_t) size > psf->header.len && psf_bump_header_allocation (psf, size))
						return count ;

					count += size ;
					while (size)
					{	psf->header.ptr [psf->header.indx] = 0 ;
						psf->header.indx ++ ;
						size -- ;
						} ;
					break ;

			case 'h' :
					bindata = va_arg (argptr, void *) ;
					memcpy (&(psf->header.ptr [psf->header.indx]), bindata, 16) ;
					psf->header.indx += 16 ;
					count += 16 ;
					break ;

			case 'j' :	/* Jump forwards/backwards by specified amount. */
					size = va_arg (argptr, size_t) ;

					if (psf->header.indx + (sf_count_t) size > psf->header.len && psf_bump_header_allocation (psf, size))
						return count ;

					psf->header.indx += size ;
					count += size ;
					break ;

			case 'o' :	/* Jump to specified offset. */
					size = va_arg (argptr, size_t) ;

					if ((sf_count_t) size >= psf->header.len && psf_bump_header_allocation (psf, size))
						return count ;

					psf->header.indx = size ;
					break ;

			default :
				psf_log_printf (psf, "*** Invalid format specifier `%c'\n", c) ;
				psf->error = SFE_INTERNAL ;
				break ;
			} ;
		} ;

	va_end (argptr) ;
	return count ;
} /* psf_binheader_writef */