Commit message: Don't treat the packet length as unsigned.

The scanf family of functions are as annoyingly bad at handling unsigned
numbers as strtoul() is - both of them are perfectly willing to accept a
value beginning with a negative sign as an unsigned value.  When using
strtoul(), you can compensate for this by explicitly checking for a '-'
as the first character of the string, but you can't do that with
sscanf().

So revert to having pkt_len be signed, and scanning it with %d, but
check for a negative value and fail if we see a negative value.

Bug: 12395
Change-Id: I43b458a73b0934e9a5c2c89d34eac5a8f21a7455
Reviewed-on: https://code.wireshark.org/review/15223
Reviewed-by: Guy Harris <guy@alum.mit.edu>
CVE: CVE-2016-5356
CWE: CWE-119
Vulnerability description: wiretap/cosine.c in the CoSine file parser in Wireshark 1.12.x before 1.12.12 and 2.x before 2.0.4 mishandles sscanf unsigned-integer processing, which allows remote attackers to cause a denial of service (application crash) via a crafted file.

Project: wireshark
File: cosine.c

wtap_open_return_val cosine_open(wtap *wth, int *err, gchar **err_info)
{
	/* Look for CoSine header */
	if (!cosine_check_file_type(wth, err, err_info)) {
		if (*err != 0 && *err != WTAP_ERR_SHORT_READ)
			return WTAP_OPEN_ERROR;
		return WTAP_OPEN_NOT_MINE;
	}

	if (file_seek(wth->fh, 0L, SEEK_SET, err) == -1)	/* rewind */
		return WTAP_OPEN_ERROR;

	wth->file_encap = WTAP_ENCAP_COSINE;
	wth->file_type_subtype = WTAP_FILE_TYPE_SUBTYPE_COSINE;
	wth->snapshot_length = 0; /* not known */
	wth->subtype_read = cosine_read;
	wth->subtype_seek_read = cosine_seek_read;
	wth->file_tsprec = WTAP_TSPREC_CSEC;

	return WTAP_OPEN_MINE;
}

Project: wireshark
File: cosine.c

static gboolean cosine_read(wtap *wth, int *err, gchar **err_info,
    gint64 *data_offset)
{
	gint64	offset;
	char	line[COSINE_LINE_LENGTH];

	/* Find the next packet */
	offset = cosine_seek_next_packet(wth, err, err_info, line);
	if (offset < 0)
		return FALSE;
	*data_offset = offset;

	/* Parse the header and convert the ASCII hex dump to binary data */
	return parse_cosine_packet(wth->fh, &wth->phdr, wth->frame_buffer,
	    line, err, err_info);
}

Project: wireshark
File: cosine.c

cosine_seek_read(wtap *wth, gint64 seek_off, struct wtap_pkthdr *phdr,
	Buffer *buf, int *err, gchar **err_info)
{
	char	line[COSINE_LINE_LENGTH];

	if (file_seek(wth->random_fh, seek_off, SEEK_SET, err) == -1)
		return FALSE;

	if (file_gets(line, COSINE_LINE_LENGTH, wth->random_fh) == NULL) {
		*err = file_error(wth->random_fh, err_info);
		if (*err == 0) {
			*err = WTAP_ERR_SHORT_READ;
		}
		return FALSE;
	}

	/* Parse the header and convert the ASCII hex dump to binary data */
	return parse_cosine_packet(wth->random_fh, phdr, buf, line, err,
	    err_info);
}

Project: wireshark
File: cosine.c

static gboolean empty_line(const gchar *line)
{
	while (*line) {
		if (g_ascii_isspace(*line)) {
			line++;
			continue;
		} else {
			break;
		}
	}
	if (*line == '\0')
		return TRUE;
	else
		return FALSE;
}

Project: wireshark
File: cosine.c

parse_single_hex_dump_line(char* rec, guint8 *buf, guint byte_offset)
{
	int num_items_scanned, i;
	unsigned int bytes[16];

	num_items_scanned = sscanf(rec, "%02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x %02x",
			       &bytes[0], &bytes[1], &bytes[2], &bytes[3],
			       &bytes[4], &bytes[5], &bytes[6], &bytes[7],
			       &bytes[8], &bytes[9], &bytes[10], &bytes[11],
			       &bytes[12], &bytes[13], &bytes[14], &bytes[15]);
	if (num_items_scanned == 0)
		return -1;

	if (num_items_scanned > 16)
		num_items_scanned = 16;

	for (i=0; i<num_items_scanned; i++) {
		buf[byte_offset + i] = (guint8)bytes[i];
	}

	return num_items_scanned;
}