Commit message: handshake: set a maximum number of warning messages that can be received per handshake

That is to avoid DoS due to the assymetry of cost of sending an alert vs the cost
of processing.
CVE: CVE-2016-8610
CWE: CWE-400
Vulnerability description: A denial of service flaw was found in OpenSSL 0.9.8, 1.0.1, 1.0.2 through 1.0.2h, and 1.1.0 in the way the TLS/SSL protocol defined processing of ALERT packets during a connection handshake. A remote attacker could use this flaw to make a TLS/SSL server consume an excessive amount of CPU and fail to accept connections from other clients.

Project: gnutls
File: state.c

gnutls_handshake_set_hook_function(gnutls_session_t session,
				   unsigned int htype,
				   int post,
				   gnutls_handshake_hook_func func)
{
	session->internals.h_hook = func;
	session->internals.h_type = htype;
	session->internals.h_post = post;
}

Project: gnutls
File: state.c

gnutls_openpgp_send_cert(gnutls_session_t session,
			 gnutls_openpgp_crt_status_t status)
{
	session->internals.pgp_fingerprint = status;
}

Project: gnutls
File: state.c

int gnutls_init(gnutls_session_t * session, unsigned int flags)
{
	int ret;
	record_parameters_st *epoch;
	
	FAIL_IF_LIB_ERROR;

	*session = gnutls_calloc(1, sizeof(struct gnutls_session_int));
	if (*session == NULL)
		return GNUTLS_E_MEMORY_ERROR;

	ret = _gnutls_epoch_alloc(*session, 0, &epoch);
	if (ret < 0) {
		gnutls_assert();
		return GNUTLS_E_MEMORY_ERROR;
	}

	/* Set all NULL algos on epoch 0 */
	_gnutls_epoch_set_null_algos(*session, epoch);

	(*session)->security_parameters.epoch_next = 1;

	(*session)->security_parameters.entity =
	    (flags & GNUTLS_SERVER ? GNUTLS_SERVER : GNUTLS_CLIENT);

	/* the default certificate type for TLS */
	(*session)->security_parameters.cert_type = DEFAULT_CERT_TYPE;

	/* Initialize buffers */
	_gnutls_buffer_init(&(*session)->internals.handshake_hash_buffer);
	_gnutls_buffer_init(&(*session)->internals.hb_remote_data);
	_gnutls_buffer_init(&(*session)->internals.hb_local_data);
	_gnutls_buffer_init(&(*session)->internals.record_presend_buffer);

	_mbuffer_head_init(&(*session)->internals.record_buffer);
	_mbuffer_head_init(&(*session)->internals.record_send_buffer);
	_mbuffer_head_init(&(*session)->internals.record_recv_buffer);

	_mbuffer_head_init(&(*session)->internals.handshake_send_buffer);
	_gnutls_handshake_recv_buffer_init(*session);

	(*session)->internals.expire_time = DEFAULT_EXPIRE_TIME;	/* one hour default */

	gnutls_handshake_set_max_packet_length((*session),
					       MAX_HANDSHAKE_PACKET_SIZE);

	/* set the socket pointers to -1;
	 */
	(*session)->internals.transport_recv_ptr =
	    (gnutls_transport_ptr_t) - 1;
	(*session)->internals.transport_send_ptr =
	    (gnutls_transport_ptr_t) - 1;

	/* set the default maximum record size for TLS
	 */
	(*session)->security_parameters.max_record_recv_size =
	    DEFAULT_MAX_RECORD_SIZE;
	(*session)->security_parameters.max_record_send_size =
	    DEFAULT_MAX_RECORD_SIZE;

	/* everything else not initialized here is initialized
	 * as NULL or 0. This is why calloc is used.
	 */

	_gnutls_handshake_internal_state_init(*session);

	(*session)->internals.extensions_sent_size = 0;

	/* emulate old gnutls behavior for old applications that do not use the priority_*
	 * functions.
	 */
	(*session)->internals.priorities.sr = SR_PARTIAL;

#ifdef HAVE_WRITEV
#ifdef MSG_NOSIGNAL
	if (flags & GNUTLS_NO_SIGNAL)
		gnutls_transport_set_vec_push_function(*session, system_writev_nosignal);
	else
#endif
		gnutls_transport_set_vec_push_function(*session, system_writev);
#else
	gnutls_transport_set_push_function(*session, system_write);
#endif
	(*session)->internals.pull_timeout_func = gnutls_system_recv_timeout;
	(*session)->internals.pull_func = system_read;
	(*session)->internals.errno_func = system_errno;

	/* heartbeat timeouts */
	(*session)->internals.hb_retrans_timeout_ms = 1000;
	(*session)->internals.hb_total_timeout_ms = 60000;

	if (flags & GNUTLS_DATAGRAM) {
		(*session)->internals.dtls.mtu = DTLS_DEFAULT_MTU;
		(*session)->internals.transport = GNUTLS_DGRAM;

		gnutls_dtls_set_timeouts(*session, DTLS_RETRANS_TIMEOUT, 60000);
	} else {
		(*session)->internals.transport = GNUTLS_STREAM;
	}

	/* Enable useful extensions */
	if ((flags & GNUTLS_CLIENT) && !(flags & GNUTLS_NO_EXTENSIONS)) {
#ifdef ENABLE_SESSION_TICKETS
		gnutls_session_ticket_enable_client(*session);
#endif
#ifdef ENABLE_OCSP
		gnutls_ocsp_status_request_enable_client(*session, NULL, 0,
							 NULL);
#endif
	}

	(*session)->internals.flags = flags;

	return 0;
}

Project: gnutls
File: state.c

gnutls_kx_algorithm_t gnutls_kx_get(gnutls_session_t session)
{
	return session->security_parameters.kx_algorithm;
}

Project: gnutls
File: state.c

gnutls_certificate_type_get(gnutls_session_t session)
{
	return session->security_parameters.cert_type;
}

Project: gnutls
File: state.c

gnutls_handshake_set_private_extensions(gnutls_session_t session,
					int allow)
{
	session->internals.enable_private = allow;
}

Project: gnutls
File: state.c

_gnutls_session_cert_type_set(gnutls_session_t session,
			      gnutls_certificate_type_t ct)
{
	_gnutls_handshake_log
	    ("HSK[%p]: Selected certificate type %s (%d)\n", session,
	     gnutls_certificate_type_get_name(ct), ct);
	session->security_parameters.cert_type = ct;
}

Project: gnutls
File: state.c

_gnutls_dh_set_group(gnutls_session_t session, bigint_t gen,
		     bigint_t prime)
{
	dh_info_st *dh;
	int ret;

	switch (gnutls_auth_get_type(session)) {
	case GNUTLS_CRD_ANON:
		{
			anon_auth_info_t info;
			info = _gnutls_get_auth_info(session, GNUTLS_CRD_ANON);
			if (info == NULL)
				return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);

			dh = &info->dh;
			break;
		}
	case GNUTLS_CRD_PSK:
		{
			psk_auth_info_t info;
			info = _gnutls_get_auth_info(session, GNUTLS_CRD_PSK);
			if (info == NULL)
				return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);

			dh = &info->dh;
			break;
		}
	case GNUTLS_CRD_CERTIFICATE:
		{
			cert_auth_info_t info;

			info = _gnutls_get_auth_info(session, GNUTLS_CRD_CERTIFICATE);
			if (info == NULL)
				return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);

			dh = &info->dh;
			break;
		}
	default:
		return gnutls_assert_val(GNUTLS_E_INTERNAL_ERROR);
	}

	if (dh->prime.data)
		_gnutls_free_datum(&dh->prime);

	if (dh->generator.data)
		_gnutls_free_datum(&dh->generator);

	/* prime
	 */
	ret = _gnutls_mpi_dprint_lz(prime, &dh->prime);
	if (ret < 0) {
		gnutls_assert();
		return ret;
	}

	/* generator
	 */
	ret = _gnutls_mpi_dprint_lz(gen, &dh->generator);
	if (ret < 0) {
		gnutls_assert();
		_gnutls_free_datum(&dh->prime);
		return ret;
	}

	return 0;
}

Project: gnutls
File: state.c

void _gnutls_handshake_internal_state_clear(gnutls_session_t session)
{
	_gnutls_handshake_internal_state_init(session);

	deinit_internal_params(session);
	deinit_keys(session);

	_gnutls_epoch_gc(session);

	session->internals.handshake_endtime = 0;
	session->internals.handshake_in_progress = 0;

	session->internals.tfo.connect_addrlen = 0;
	session->internals.tfo.connect_only = 0;
}

Project: gnutls
File: state.c

int _gnutls_session_is_psk(gnutls_session_t session)
{
	gnutls_kx_algorithm_t kx;

	kx = _gnutls_cipher_suite_get_kx_algo(session->security_parameters.
					      cipher_suite);
	if (kx == GNUTLS_KX_PSK || kx == GNUTLS_KX_DHE_PSK
	    || kx == GNUTLS_KX_RSA_PSK)
		return 1;

	return 0;
}

Project: gnutls
File: state.c

gnutls_compression_get(gnutls_session_t session)
{
	record_parameters_st *record_params;
	int ret;

	ret =
	    _gnutls_epoch_get(session, EPOCH_READ_CURRENT, &record_params);
	if (ret < 0)
		return gnutls_assert_val(GNUTLS_COMP_NULL);

	return record_params->compression_algorithm;
}

Project: gnutls
File: state.c

void gnutls_deinit(gnutls_session_t session)
{
	unsigned int i;

	if (session == NULL)
		return;

	/* remove auth info firstly */
	_gnutls_free_auth_info(session);

	_gnutls_handshake_internal_state_clear(session);
	_gnutls_handshake_io_buffer_clear(session);
	_gnutls_ext_free_session_data(session);

	for (i = 0; i < MAX_EPOCH_INDEX; i++)
		if (session->record_parameters[i] != NULL) {
			_gnutls_epoch_free(session,
					   session->record_parameters[i]);
			session->record_parameters[i] = NULL;
		}

	_gnutls_buffer_clear(&session->internals.handshake_hash_buffer);
	_gnutls_buffer_clear(&session->internals.hb_remote_data);
	_gnutls_buffer_clear(&session->internals.hb_local_data);
	_gnutls_buffer_clear(&session->internals.record_presend_buffer);

	_mbuffer_head_clear(&session->internals.record_buffer);
	_mbuffer_head_clear(&session->internals.record_recv_buffer);
	_mbuffer_head_clear(&session->internals.record_send_buffer);

	_gnutls_free_datum(&session->internals.resumption_data);

	gnutls_free(session->internals.rexts);

	gnutls_free(session->internals.rsup);

	gnutls_credentials_clear(session);
	_gnutls_selected_certs_deinit(session);

	gnutls_free(session);
}

Project: gnutls
File: state.c

gnutls_record_set_state(gnutls_session_t session,
			unsigned read,
			unsigned char seq_number[8])
{
	record_parameters_st *record_params;
	record_state_st *record_state;
	int epoch, ret;

	if (read)
		epoch = EPOCH_READ_CURRENT;
	else
		epoch = EPOCH_WRITE_CURRENT;

	ret = _gnutls_epoch_get(session, epoch, &record_params);
	if (ret < 0)
		return gnutls_assert_val(ret);

	if (!record_params->initialized)
		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);

	if (read)
		record_state = &record_params->read;
	else
		record_state = &record_params->write;

	memcpy(UINT64DATA(record_state->sequence_number), seq_number, 8);

	if (IS_DTLS(session)) {
		_dtls_reset_window(record_params);
	}

	return 0;
}

Project: gnutls
File: state.c

int _gnutls_openpgp_send_fingerprint(gnutls_session_t session)
{
	return session->internals.pgp_fingerprint;
}

Project: gnutls
File: state.c

void gnutls_session_enable_compatibility_mode(gnutls_session_t session)
{
	ENABLE_COMPAT(&session->internals.priorities);
}

Project: gnutls
File: state.c

static void deinit_keys(gnutls_session_t session)
{
	gnutls_pk_params_release(&session->key.ecdh_params);
	gnutls_pk_params_release(&session->key.dh_params);
	zrelease_temp_mpi_key(&session->key.ecdh_x);
	zrelease_temp_mpi_key(&session->key.ecdh_y);
	_gnutls_free_temp_key_datum(&session->key.ecdhx);

	zrelease_temp_mpi_key(&session->key.client_Y);

	/* SRP */
	zrelease_temp_mpi_key(&session->key.srp_p);
	zrelease_temp_mpi_key(&session->key.srp_g);
	zrelease_temp_mpi_key(&session->key.srp_key);

	zrelease_temp_mpi_key(&session->key.u);
	zrelease_temp_mpi_key(&session->key.a);
	zrelease_temp_mpi_key(&session->key.x);
	zrelease_temp_mpi_key(&session->key.A);
	zrelease_temp_mpi_key(&session->key.B);
	zrelease_temp_mpi_key(&session->key.b);

	_gnutls_free_temp_key_datum(&session->key.key);
	_gnutls_free_temp_key_datum(&session->key.key);
}

Project: gnutls
File: state.c

gnutls_ecc_curve_t gnutls_ecc_curve_get(gnutls_session_t session)
{
	return _gnutls_session_ecc_curve_get(session);
}

Project: gnutls
File: state.c

unsigned gnutls_session_get_flags(gnutls_session_t session)
{
	unsigned flags = 0;

	if (gnutls_safe_renegotiation_status(session))
		flags |= GNUTLS_SFLAGS_SAFE_RENEGOTIATION;
	if (gnutls_session_ext_master_secret_status(session))
		flags |= GNUTLS_SFLAGS_EXT_MASTER_SECRET;
	if (gnutls_session_etm_status(session))
		flags |= GNUTLS_SFLAGS_ETM;
	if (gnutls_heartbeat_allowed(session, GNUTLS_HB_LOCAL_ALLOWED_TO_SEND))
		flags |= GNUTLS_SFLAGS_HB_LOCAL_SEND;
	if (gnutls_heartbeat_allowed(session, GNUTLS_HB_PEER_ALLOWED_TO_SEND))
		flags |= GNUTLS_SFLAGS_HB_PEER_SEND;
	if (session->internals.false_start_used)
		flags |= GNUTLS_SFLAGS_FALSE_START;

	return flags;
}

Project: gnutls
File: state.c

_gnutls_session_cert_type_supported(gnutls_session_t session,
				    gnutls_certificate_type_t cert_type)
{
	unsigned i;
	unsigned cert_found = 0;
	gnutls_certificate_credentials_t cred;

	if (session->security_parameters.entity == GNUTLS_SERVER) {
		cred = (gnutls_certificate_credentials_t)
		    _gnutls_get_cred(session, GNUTLS_CRD_CERTIFICATE);

		if (cred == NULL)
			return GNUTLS_E_UNSUPPORTED_CERTIFICATE_TYPE;

		if (cred->get_cert_callback == NULL && cred->get_cert_callback2 == NULL) {
			for (i = 0; i < cred->ncerts; i++) {
				if (cred->certs[i].cert_list[0].type ==
				    cert_type) {
					cert_found = 1;
					break;
				}
			}

			if (cert_found == 0)
				/* no certificate is of that type.
				 */
				return
				    GNUTLS_E_UNSUPPORTED_CERTIFICATE_TYPE;
		}
	}

	if (session->internals.priorities.cert_type.algorithms == 0
	    && cert_type == DEFAULT_CERT_TYPE)
		return 0;

	for (i = 0; i < session->internals.priorities.cert_type.algorithms;
	     i++) {
		if (session->internals.priorities.cert_type.priority[i] ==
		    cert_type) {
			return 0;	/* ok */
		}
	}

	return GNUTLS_E_UNSUPPORTED_CERTIFICATE_TYPE;
}

Project: gnutls
File: state.c

static void _gnutls_handshake_internal_state_init(gnutls_session_t session)
{
	/* by default no selected certificate */
	session->internals.adv_version_major = 0;
	session->internals.adv_version_minor = 0;
	session->internals.direction = 0;

	/* use out of band data for the last
	 * handshake messages received.
	 */
	session->internals.last_handshake_in = -1;
	session->internals.last_handshake_out = -1;

	session->internals.resumable = RESUME_TRUE;

	session->internals.handshake_suspicious_loops = 0;
	session->internals.dtls.hsk_read_seq = 0;
	session->internals.dtls.hsk_write_seq = 0;
}

Project: gnutls
File: state.c

inline static void deinit_internal_params(gnutls_session_t session)
{
#if defined(ENABLE_DHE) || defined(ENABLE_ANON)
	if (session->internals.params.free_dh_params)
		gnutls_dh_params_deinit(session->internals.params.
					dh_params);
#endif

	_gnutls_handshake_hash_buffers_clear(session);

	memset(&session->internals.params, 0,
	       sizeof(session->internals.params));
}

Project: gnutls
File: state.c

int gnutls_session_is_resumed(gnutls_session_t session)
{
	if (session->security_parameters.entity == GNUTLS_CLIENT) {
		if (session->security_parameters.session_id_size > 0 &&
		    session->security_parameters.session_id_size ==
		    session->internals.resumed_security_parameters.
		    session_id_size
		    && memcmp(session->security_parameters.session_id,
			      session->
			      internals.resumed_security_parameters.
			      session_id,
			      session->security_parameters.
			      session_id_size) == 0)
			return 1;
	} else {
		if (session->internals.resumed != RESUME_FALSE)
			return 1;
	}

	return 0;
}

Project: gnutls
File: state.c

gnutls_record_get_state(gnutls_session_t session,
			unsigned read,
			gnutls_datum_t *mac_key,
			gnutls_datum_t *IV,
			gnutls_datum_t *cipher_key,
			unsigned char seq_number[8])
{
	record_parameters_st *record_params;
	record_state_st *record_state;
	unsigned int epoch;
	int ret;

	if (read)
		epoch = EPOCH_READ_CURRENT;
	else
		epoch = EPOCH_WRITE_CURRENT;

	ret = _gnutls_epoch_get(session, epoch, &record_params);
	if (ret < 0)
		return gnutls_assert_val(ret);

	if (!record_params->initialized)
		return gnutls_assert_val(GNUTLS_E_INVALID_REQUEST);

	if (read)
		record_state = &record_params->read;
	else
		record_state = &record_params->write;

	if (mac_key)
		memcpy(mac_key, &record_state->mac_secret, sizeof(gnutls_datum_t));
	if (IV)
		memcpy(IV, &record_state->IV, sizeof(gnutls_datum_t));
	if (cipher_key)
		memcpy(cipher_key, &record_state->key, sizeof(gnutls_datum_t));
	if (seq_number)
		memcpy(seq_number, UINT64DATA(record_state->sequence_number), 8);
	return 0;
}

Project: gnutls
File: state.c

int _gnutls_session_is_ecc(gnutls_session_t session)
{
	gnutls_kx_algorithm_t kx;

	/* We get the key exchange algorithm through the ciphersuite because
	 * the negotiated key exchange might not have been set yet.
	 */
	kx = _gnutls_cipher_suite_get_kx_algo(session->security_parameters.
					      cipher_suite);

	return _gnutls_kx_is_ecc(kx);
}

Project: gnutls
File: state.c

_gnutls_session_ecc_curve_set(gnutls_session_t session,
			      gnutls_ecc_curve_t c)
{
	_gnutls_handshake_log("HSK[%p]: Selected ECC curve %s (%d)\n",
			      session, gnutls_ecc_curve_get_name(c), c);
	session->security_parameters.ecc_curve = c;
}