Commit message: delta: fix out-of-bounds read of delta

When computing the offset and length of the delta base, we repeatedly
increment the `delta` pointer without checking whether we have advanced
past its end already, which can thus result in an out-of-bounds read.
Fix this by repeatedly checking whether we have reached the end. Add a
test which would cause Valgrind to produce an error.

Reported-by: Riccardo Schirone <rschiron@redhat.com>
Test-provided-by: Riccardo Schirone <rschiron@redhat.com>
CVE: CVE-2018-10888
CWE: CWE-125
Vulnerability description: A flaw was found in libgit2 before version 0.27.3. A missing check in git_delta_apply function in delta.c file, may lead to an out-of-bound read while reading a binary delta file. An attacker may use this flaw to cause a Denial of Service.

Project: libgit2
File: delta.c

int git_delta_create_from_index(
	void **out,
	size_t *out_len,
	const struct git_delta_index *index,
	const void *trg_buf,
	size_t trg_size,
	size_t max_size)
{
	unsigned int i, bufpos, bufsize, moff, msize, val;
	int inscnt;
	const unsigned char *ref_data, *ref_top, *data, *top;
	unsigned char *buf;

	*out = NULL;
	*out_len = 0;

	if (!trg_buf || !trg_size)
		return 0;

	bufpos = 0;
	bufsize = 8192;
	if (max_size && bufsize >= max_size)
		bufsize = (unsigned int)(max_size + MAX_OP_SIZE + 1);
	buf = git__malloc(bufsize);
	GITERR_CHECK_ALLOC(buf);

	/* store reference buffer size */
	i = index->src_size;
	while (i >= 0x80) {
		buf[bufpos++] = i | 0x80;
		i >>= 7;
	}
	buf[bufpos++] = i;

	/* store target buffer size */
	i = trg_size;
	while (i >= 0x80) {
		buf[bufpos++] = i | 0x80;
		i >>= 7;
	}
	buf[bufpos++] = i;

	ref_data = index->src_buf;
	ref_top = ref_data + index->src_size;
	data = trg_buf;
	top = (const unsigned char *) trg_buf + trg_size;

	bufpos++;
	val = 0;
	for (i = 0; i < RABIN_WINDOW && data < top; i++, data++) {
		buf[bufpos++] = *data;
		val = ((val << 8) | *data) ^ T[val >> RABIN_SHIFT];
	}
	inscnt = i;

	moff = 0;
	msize = 0;
	while (data < top) {
		if (msize < 4096) {
			struct index_entry *entry;
			val ^= U[data[-RABIN_WINDOW]];
			val = ((val << 8) | *data) ^ T[val >> RABIN_SHIFT];
			i = val & index->hash_mask;
			for (entry = index->hash[i]; entry; entry = entry->next) {
				const unsigned char *ref = entry->ptr;
				const unsigned char *src = data;
				unsigned int ref_size = (unsigned int)(ref_top - ref);
				if (entry->val != val)
					continue;
				if (ref_size > (unsigned int)(top - src))
					ref_size = (unsigned int)(top - src);
				if (ref_size <= msize)
					break;
				while (ref_size-- && *src++ == *ref)
					ref++;
				if (msize < (unsigned int)(ref - entry->ptr)) {
					/* this is our best match so far */
					msize = (unsigned int)(ref - entry->ptr);
					moff = (unsigned int)(entry->ptr - ref_data);
					if (msize >= 4096) /* good enough */
						break;
				}
			}
		}

		if (msize < 4) {
			if (!inscnt)
				bufpos++;
			buf[bufpos++] = *data++;
			inscnt++;
			if (inscnt == 0x7f) {
				buf[bufpos - inscnt - 1] = inscnt;
				inscnt = 0;
			}
			msize = 0;
		} else {
			unsigned int left;
			unsigned char *op;

			if (inscnt) {
				while (moff && ref_data[moff-1] == data[-1]) {
					/* we can match one byte back */
					msize++;
					moff--;
					data--;
					bufpos--;
					if (--inscnt)
						continue;
					bufpos--;  /* remove count slot */
					inscnt--;  /* make it -1 */
					break;
				}
				buf[bufpos - inscnt - 1] = inscnt;
				inscnt = 0;
			}

			/* A copy op is currently limited to 64KB (pack v2) */
			left = (msize < 0x10000) ? 0 : (msize - 0x10000);
			msize -= left;

			op = buf + bufpos++;
			i = 0x80;

			if (moff & 0x000000ff)
				buf[bufpos++] = moff >> 0,  i |= 0x01;
			if (moff & 0x0000ff00)
				buf[bufpos++] = moff >> 8,  i |= 0x02;
			if (moff & 0x00ff0000)
				buf[bufpos++] = moff >> 16, i |= 0x04;
			if (moff & 0xff000000)
				buf[bufpos++] = moff >> 24, i |= 0x08;

			if (msize & 0x00ff)
				buf[bufpos++] = msize >> 0, i |= 0x10;
			if (msize & 0xff00)
				buf[bufpos++] = msize >> 8, i |= 0x20;

			*op = i;

			data += msize;
			moff += msize;
			msize = left;

			if (msize < 4096) {
				int j;
				val = 0;
				for (j = -RABIN_WINDOW; j < 0; j++)
					val = ((val << 8) | data[j])
					      ^ T[val >> RABIN_SHIFT];
			}
		}

		if (bufpos >= bufsize - MAX_OP_SIZE) {
			void *tmp = buf;
			bufsize = bufsize * 3 / 2;
			if (max_size && bufsize >= max_size)
				bufsize = max_size + MAX_OP_SIZE + 1;
			if (max_size && bufpos > max_size)
				break;
			buf = git__realloc(buf, bufsize);
			if (!buf) {
				git__free(tmp);
				return -1;
			}
		}
	}

	if (inscnt)
		buf[bufpos - inscnt - 1] = inscnt;

	if (max_size && bufpos > max_size) {
		giterr_set(GITERR_NOMEMORY, "delta would be larger than maximum size");
		git__free(buf);
		return GIT_EBUFS;
	}

	*out_len = bufpos;
	*out = buf;
	return 0;
}

Project: libgit2
File: delta.c

void git_delta_index_free(git_delta_index *index)
{
	git__free(index);
}

Project: libgit2
File: delta.c

int git_delta_index_init(
	git_delta_index **out, const void *buf, size_t bufsize)
{
	unsigned int i, hsize, hmask, entries, prev_val, *hash_count;
	const unsigned char *data, *buffer = buf;
	struct git_delta_index *index;
	struct index_entry *entry, **hash;
	void *mem;
	unsigned long memsize;

	*out = NULL;

	if (!buf || !bufsize)
		return 0;

	/* Determine index hash size.  Note that indexing skips the
	   first byte to allow for optimizing the rabin polynomial
	   initialization in create_delta(). */
	entries = (unsigned int)(bufsize - 1) / RABIN_WINDOW;
	if (bufsize >= 0xffffffffUL) {
		/*
		 * Current delta format can't encode offsets into
		 * reference buffer with more than 32 bits.
		 */
		entries = 0xfffffffeU / RABIN_WINDOW;
	}
	hsize = entries / 4;
	for (i = 4; i < 31 && (1u << i) < hsize; i++);
	hsize = 1 << i;
	hmask = hsize - 1;

	if (lookup_index_alloc(&mem, &memsize, entries, hsize) < 0)
		return -1;

	index = mem;
	mem = index->hash;
	hash = mem;
	mem = hash + hsize;
	entry = mem;

	index->memsize = memsize;
	index->src_buf = buf;
	index->src_size = bufsize;
	index->hash_mask = hmask;
	memset(hash, 0, hsize * sizeof(*hash));

	/* allocate an array to count hash entries */
	hash_count = git__calloc(hsize, sizeof(*hash_count));
	if (!hash_count) {
		git__free(index);
		return -1;
	}

	/* then populate the index */
	prev_val = ~0;
	for (data = buffer + entries * RABIN_WINDOW - RABIN_WINDOW;
	     data >= buffer;
	     data -= RABIN_WINDOW) {
		unsigned int val = 0;
		for (i = 1; i <= RABIN_WINDOW; i++)
			val = ((val << 8) | data[i]) ^ T[val >> RABIN_SHIFT];
		if (val == prev_val) {
			/* keep the lowest of consecutive identical blocks */
			entry[-1].ptr = data + RABIN_WINDOW;
		} else {
			prev_val = val;
			i = val & hmask;
			entry->ptr = data + RABIN_WINDOW;
			entry->val = val;
			entry->next = hash[i];
			hash[i] = entry++;
			hash_count[i]++;
		}
	}

	/*
	 * Determine a limit on the number of entries in the same hash
	 * bucket.  This guard us against patological data sets causing
	 * really bad hash distribution with most entries in the same hash
	 * bucket that would bring us to O(m*n) computing costs (m and n
	 * corresponding to reference and target buffer sizes).
	 *
	 * Make sure none of the hash buckets has more entries than
	 * we're willing to test.  Otherwise we cull the entry list
	 * uniformly to still preserve a good repartition across
	 * the reference buffer.
	 */
	for (i = 0; i < hsize; i++) {
		if (hash_count[i] < HASH_LIMIT)
			continue;

		entry = hash[i];
		do {
			struct index_entry *keep = entry;
			int skip = hash_count[i] / HASH_LIMIT / 2;
			do {
				entry = entry->next;
			} while(--skip && entry);
			keep->next = entry;
		} while (entry);
	}
	git__free(hash_count);

	*out = index;
	return 0;
}

Project: libgit2
File: delta.c

size_t git_delta_index_size(git_delta_index *index)
{
	assert(index);

	return index->memsize;
}

Project: libgit2
File: delta.c

int git_delta_read_header(
	size_t *base_out,
	size_t *result_out,
	const unsigned char *delta,
	size_t delta_len)
{
	const unsigned char *delta_end = delta + delta_len;
	if ((hdr_sz(base_out, &delta, delta_end) < 0) ||
		(hdr_sz(result_out, &delta, delta_end) < 0))
		return -1;
	return 0;
}

Project: libgit2
File: delta.c

int git_delta_read_header_fromstream(
	size_t *base_sz, size_t *res_sz, git_packfile_stream *stream)
{
	static const size_t buffer_len = DELTA_HEADER_BUFFER_LEN;
	unsigned char buffer[DELTA_HEADER_BUFFER_LEN];
	const unsigned char *delta, *delta_end;
	size_t len;
	ssize_t read;

	len = read = 0;
	while (len < buffer_len) {
		read = git_packfile_stream_read(stream, &buffer[len], buffer_len - len);

		if (read == 0)
			break;

		if (read == GIT_EBUFS)
			continue;

		len += read;
	}

	delta = buffer;
	delta_end = delta + len;
	if ((hdr_sz(base_sz, &delta, delta_end) < 0) ||
		(hdr_sz(res_sz, &delta, delta_end) < 0))
		return -1;

	return 0;
}

Project: libgit2
File: delta.c

static int hdr_sz(
	size_t *size,
	const unsigned char **delta,
	const unsigned char *end)
{
	const unsigned char *d = *delta;
	size_t r = 0;
	unsigned int c, shift = 0;

	do {
		if (d == end) {
			giterr_set(GITERR_INVALID, "truncated delta");
			return -1;
		}

		c = *d++;
		r |= (c & 0x7f) << shift;
		shift += 7;
	} while (c & 0x80);
	*delta = d;
	*size = r;
	return 0;
}

Project: libgit2
File: delta.c

static int lookup_index_alloc(
	void **out, unsigned long *out_len, size_t entries, size_t hash_count)
{
	size_t entries_len, hash_len, index_len;

	GITERR_CHECK_ALLOC_MULTIPLY(&entries_len, entries, sizeof(struct index_entry));
	GITERR_CHECK_ALLOC_MULTIPLY(&hash_len, hash_count, sizeof(struct index_entry *));

	GITERR_CHECK_ALLOC_ADD(&index_len, sizeof(struct git_delta_index), entries_len);
	GITERR_CHECK_ALLOC_ADD(&index_len, index_len, hash_len);

	if (!git__is_ulong(index_len)) {
		giterr_set(GITERR_NOMEMORY, "overly large delta");
		return -1;
	}

	*out = git__malloc(index_len);
	GITERR_CHECK_ALLOC(*out);

	*out_len = index_len;
	return 0;
}

Project: libgit2
File: apply.c

void test_delta_apply__read_at_off(void)
{
	unsigned char base[16] = { 0 }, delta[] = { 0x10, 0x10, 0xff, 0xff, 0xff, 0xff, 0xff, 0x10, 0x00, 0x00 };
	void *out;
	size_t outlen;
 
 	cl_git_fail(git_delta_apply(&out, &outlen, base, sizeof(base), delta, sizeof(delta)));
 }