mlx5_tx_mseg_memcpy(uint8_t *pdst,
		    struct mlx5_txq_local *__rte_restrict loc,
		    unsigned int len,
		    unsigned int must,
		    unsigned int olx __rte_unused)
{
	struct rte_mbuf *mbuf;
	unsigned int part, dlen, copy = 0;
	uint8_t *psrc;

	MLX5_ASSERT(len);
	do {
		/* Allow zero length packets, must check first. */
		dlen = rte_pktmbuf_data_len(loc->mbuf);
		if (dlen <= loc->mbuf_off) {
			/* Exhausted packet, just free. */
			mbuf = loc->mbuf;
			loc->mbuf = mbuf->next;
			rte_pktmbuf_free_seg(mbuf);
			loc->mbuf_off = 0;
			MLX5_ASSERT(loc->mbuf_nseg > 1);
			MLX5_ASSERT(loc->mbuf);
			--loc->mbuf_nseg;
			if (loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) {
				unsigned int diff;

				if (copy >= must) {
					/*
					 * We already copied the minimal
					 * requested amount of data.
					 */
					return copy;
				}
				diff = must - copy;
				if (diff <= rte_pktmbuf_data_len(loc->mbuf)) {
					/*
					 * Copy only the minimal required
					 * part of the data buffer. Limit amount
					 * of data to be copied to the length of
					 * available space.
					 */
					len = RTE_MIN(len, diff);
				}
			}
			continue;
		}
		dlen -= loc->mbuf_off;
		psrc = rte_pktmbuf_mtod_offset(loc->mbuf, uint8_t *,
					       loc->mbuf_off);
		part = RTE_MIN(len, dlen);
		rte_memcpy(pdst, psrc, part);
		copy += part;
		loc->mbuf_off += part;
		len -= part;
		if (!len) {
			if (loc->mbuf_off >= rte_pktmbuf_data_len(loc->mbuf)) {
				loc->mbuf_off = 0;
				/* Exhausted packet, just free. */
				mbuf = loc->mbuf;
				loc->mbuf = mbuf->next;
				rte_pktmbuf_free_seg(mbuf);
				loc->mbuf_off = 0;
				MLX5_ASSERT(loc->mbuf_nseg >= 1);
				--loc->mbuf_nseg;
			}
			return copy;
		}
		pdst += part;
	} while (true);
}


mlx5_queue_state_modify(struct rte_eth_dev *dev,
			struct mlx5_mp_arg_queue_state_modify *sm)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	int ret = 0;

	switch (rte_eal_process_type()) {
	case RTE_PROC_PRIMARY:
		ret = mlx5_queue_state_modify_primary(dev, sm);
		break;
	case RTE_PROC_SECONDARY:
		ret = mlx5_mp_req_queue_state_modify(&priv->mp_id, sm);
		break;
	default:
		break;
	}
	return ret;
}


mlx5_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
		  struct rte_eth_rxq_info *qinfo)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_rxq_data *rxq = (*priv->rxqs)[rx_queue_id];
	struct mlx5_rxq_ctrl *rxq_ctrl =
		container_of(rxq, struct mlx5_rxq_ctrl, rxq);

	if (!rxq)
		return;
	qinfo->mp = mlx5_rxq_mprq_enabled(rxq) ?
					rxq->mprq_mp : rxq->mp;
	qinfo->conf.rx_thresh.pthresh = 0;
	qinfo->conf.rx_thresh.hthresh = 0;
	qinfo->conf.rx_thresh.wthresh = 0;
	qinfo->conf.rx_free_thresh = rxq->rq_repl_thresh;
	qinfo->conf.rx_drop_en = 1;
	qinfo->conf.rx_deferred_start = rxq_ctrl ? 0 : 1;
	qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
	qinfo->scattered_rx = dev->data->scattered_rx;
	qinfo->nb_desc = mlx5_rxq_mprq_enabled(rxq) ?
		RTE_BIT32(rxq->elts_n) * RTE_BIT32(rxq->log_strd_num) :
		RTE_BIT32(rxq->elts_n);
}


txq_ol_cksum_to_cs(struct rte_mbuf *buf)
{
	uint32_t idx;
	uint8_t is_tunnel = !!(buf->ol_flags & PKT_TX_TUNNEL_MASK);
	const uint64_t ol_flags_mask = PKT_TX_TCP_SEG | PKT_TX_L4_MASK |
				       PKT_TX_IP_CKSUM | PKT_TX_OUTER_IP_CKSUM;

	/*
	 * The index should have:
	 * bit[0] = PKT_TX_TCP_SEG
	 * bit[2:3] = PKT_TX_UDP_CKSUM, PKT_TX_TCP_CKSUM
	 * bit[4] = PKT_TX_IP_CKSUM
	 * bit[8] = PKT_TX_OUTER_IP_CKSUM
	 * bit[9] = tunnel
	 */
	idx = ((buf->ol_flags & ol_flags_mask) >> 50) | (!!is_tunnel << 9);
	return mlx5_cksum_table[idx];
}


mlx5_tx_request_completion(struct mlx5_txq_data *__rte_restrict txq,
			   struct mlx5_txq_local *__rte_restrict loc,
			   unsigned int olx)
{
	uint16_t head = txq->elts_head;
	unsigned int part;

	part = MLX5_TXOFF_CONFIG(INLINE) ?
	       0 : loc->pkts_sent - loc->pkts_copy;
	head += part;
	if ((uint16_t)(head - txq->elts_comp) >= MLX5_TX_COMP_THRESH ||
	     (MLX5_TXOFF_CONFIG(INLINE) &&
	     (uint16_t)(txq->wqe_ci - txq->wqe_comp) >= txq->wqe_thres)) {
		volatile struct mlx5_wqe *last = loc->wqe_last;

		MLX5_ASSERT(last);
		txq->elts_comp = head;
		if (MLX5_TXOFF_CONFIG(INLINE))
			txq->wqe_comp = txq->wqe_ci;
		/* Request unconditional completion on last WQE. */
		last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
					    MLX5_COMP_MODE_OFFSET);
		/* Save elts_head in dedicated free on completion queue. */
#ifdef RTE_LIBRTE_MLX5_DEBUG
		txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head |
			  (last->cseg.opcode >> 8) << 16;
#else
		txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head;
#endif
		/* A CQE slot must always be available. */
		MLX5_ASSERT((txq->cq_pi - txq->cq_ci) <= txq->cqe_s);
	}
}


mlx5_queue_state_modify_primary(struct rte_eth_dev *dev,
			const struct mlx5_mp_arg_queue_state_modify *sm)
{
	int ret;
	struct mlx5_priv *priv = dev->data->dev_private;

	if (sm->is_wq) {
		struct mlx5_rxq_data *rxq = (*priv->rxqs)[sm->queue_id];
		struct mlx5_rxq_ctrl *rxq_ctrl =
			container_of(rxq, struct mlx5_rxq_ctrl, rxq);

		ret = priv->obj_ops.rxq_obj_modify(rxq_ctrl->obj, sm->state);
		if (ret) {
			DRV_LOG(ERR, "Cannot change Rx WQ state to %u  - %s",
					sm->state, strerror(errno));
			rte_errno = errno;
			return ret;
		}
	} else {
		struct mlx5_txq_data *txq = (*priv->txqs)[sm->queue_id];
		struct mlx5_txq_ctrl *txq_ctrl =
			container_of(txq, struct mlx5_txq_ctrl, txq);

		ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj,
						   MLX5_TXQ_MOD_ERR2RDY,
						   (uint8_t)priv->dev_port);
		if (ret)
			return ret;
	}
	return 0;
}


mlx5_select_tx_function(struct rte_eth_dev *dev)
{
	struct mlx5_priv *priv = dev->data->dev_private;
	struct mlx5_dev_config *config = &priv->config;
	uint64_t tx_offloads = dev->data->dev_conf.txmode.offloads;
	unsigned int diff = 0, olx = 0, i, m;

	static_assert(MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE <=
		      MLX5_DSEG_MAX, "invalid WQE max size");
	static_assert(MLX5_WQE_CSEG_SIZE == MLX5_WSEG_SIZE,
		      "invalid WQE Control Segment size");
	static_assert(MLX5_WQE_ESEG_SIZE == MLX5_WSEG_SIZE,
		      "invalid WQE Ethernet Segment size");
	static_assert(MLX5_WQE_DSEG_SIZE == MLX5_WSEG_SIZE,
		      "invalid WQE Data Segment size");
	static_assert(MLX5_WQE_SIZE == 4 * MLX5_WSEG_SIZE,
		      "invalid WQE size");
	MLX5_ASSERT(priv);
	if (tx_offloads & DEV_TX_OFFLOAD_MULTI_SEGS) {
		/* We should support Multi-Segment Packets. */
		olx |= MLX5_TXOFF_CONFIG_MULTI;
	}
	if (tx_offloads & (DEV_TX_OFFLOAD_TCP_TSO |
			   DEV_TX_OFFLOAD_VXLAN_TNL_TSO |
			   DEV_TX_OFFLOAD_GRE_TNL_TSO |
			   DEV_TX_OFFLOAD_IP_TNL_TSO |
			   DEV_TX_OFFLOAD_UDP_TNL_TSO)) {
		/* We should support TCP Send Offload. */
		olx |= MLX5_TXOFF_CONFIG_TSO;
	}
	if (tx_offloads & (DEV_TX_OFFLOAD_IP_TNL_TSO |
			   DEV_TX_OFFLOAD_UDP_TNL_TSO |
			   DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
		/* We should support Software Parser for Tunnels. */
		olx |= MLX5_TXOFF_CONFIG_SWP;
	}
	if (tx_offloads & (DEV_TX_OFFLOAD_IPV4_CKSUM |
			   DEV_TX_OFFLOAD_UDP_CKSUM |
			   DEV_TX_OFFLOAD_TCP_CKSUM |
			   DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM)) {
		/* We should support IP/TCP/UDP Checksums. */
		olx |= MLX5_TXOFF_CONFIG_CSUM;
	}
	if (tx_offloads & DEV_TX_OFFLOAD_VLAN_INSERT) {
		/* We should support VLAN insertion. */
		olx |= MLX5_TXOFF_CONFIG_VLAN;
	}
	if (tx_offloads & DEV_TX_OFFLOAD_SEND_ON_TIMESTAMP &&
	    rte_mbuf_dynflag_lookup
			(RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL) >= 0 &&
	    rte_mbuf_dynfield_lookup
			(RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL) >= 0) {
		/* Offload configured, dynamic entities registered. */
		olx |= MLX5_TXOFF_CONFIG_TXPP;
	}
	if (priv->txqs_n && (*priv->txqs)[0]) {
		struct mlx5_txq_data *txd = (*priv->txqs)[0];

		if (txd->inlen_send) {
			/*
			 * Check the data inline requirements. Data inline
			 * is enabled on per device basis, we can check
			 * the first Tx queue only.
			 *
			 * If device does not support VLAN insertion in WQE
			 * and some queues are requested to perform VLAN
			 * insertion offload than inline must be enabled.
			 */
			olx |= MLX5_TXOFF_CONFIG_INLINE;
		}
	}
	if (config->mps == MLX5_MPW_ENHANCED &&
	    config->txq_inline_min <= 0) {
		/*
		 * The NIC supports Enhanced Multi-Packet Write
		 * and does not require minimal inline data.
		 */
		olx |= MLX5_TXOFF_CONFIG_EMPW;
	}
	if (rte_flow_dynf_metadata_avail()) {
		/* We should support Flow metadata. */
		olx |= MLX5_TXOFF_CONFIG_METADATA;
	}
	if (config->mps == MLX5_MPW) {
		/*
		 * The NIC supports Legacy Multi-Packet Write.
		 * The MLX5_TXOFF_CONFIG_MPW controls the
		 * descriptor building method in combination
		 * with MLX5_TXOFF_CONFIG_EMPW.
		 */
		if (!(olx & (MLX5_TXOFF_CONFIG_TSO |
			     MLX5_TXOFF_CONFIG_SWP |
			     MLX5_TXOFF_CONFIG_VLAN |
			     MLX5_TXOFF_CONFIG_METADATA)))
			olx |= MLX5_TXOFF_CONFIG_EMPW |
			       MLX5_TXOFF_CONFIG_MPW;
	}
	/*
	 * Scan the routines table to find the minimal
	 * satisfying routine with requested offloads.
	 */
	m = RTE_DIM(txoff_func);
	for (i = 0; i < RTE_DIM(txoff_func); i++) {
		unsigned int tmp;

		tmp = txoff_func[i].olx;
		if (tmp == olx) {
			/* Meets requested offloads exactly.*/
			m = i;
			break;
		}
		if ((tmp & olx) != olx) {
			/* Does not meet requested offloads at all. */
			continue;
		}
		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_MPW)
			/* Do not enable legacy MPW if not configured. */
			continue;
		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_EMPW)
			/* Do not enable eMPW if not configured. */
			continue;
		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_INLINE)
			/* Do not enable inlining if not configured. */
			continue;
		if ((olx ^ tmp) & MLX5_TXOFF_CONFIG_TXPP)
			/* Do not enable scheduling if not configured. */
			continue;
		/*
		 * Some routine meets the requirements.
		 * Check whether it has minimal amount
		 * of not requested offloads.
		 */
		tmp = __builtin_popcountl(tmp & ~olx);
		if (m >= RTE_DIM(txoff_func) || tmp < diff) {
			/* First or better match, save and continue. */
			m = i;
			diff = tmp;
			continue;
		}
		if (tmp == diff) {
			tmp = txoff_func[i].olx ^ txoff_func[m].olx;
			if (__builtin_ffsl(txoff_func[i].olx & ~tmp) <
			    __builtin_ffsl(txoff_func[m].olx & ~tmp)) {
				/* Lighter not requested offload. */
				m = i;
			}
		}
	}
	if (m >= RTE_DIM(txoff_func)) {
		DRV_LOG(DEBUG, "port %u has no selected Tx function"
			       " for requested offloads %04X",
				dev->data->port_id, olx);
		return NULL;
	}
	DRV_LOG(DEBUG, "port %u has selected Tx function"
		       " supporting offloads %04X/%04X",
			dev->data->port_id, olx, txoff_func[m].olx);
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MULTI)
		DRV_LOG(DEBUG, "\tMULTI (multi segment)");
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TSO)
		DRV_LOG(DEBUG, "\tTSO   (TCP send offload)");
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_SWP)
		DRV_LOG(DEBUG, "\tSWP   (software parser)");
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_CSUM)
		DRV_LOG(DEBUG, "\tCSUM  (checksum offload)");
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_INLINE)
		DRV_LOG(DEBUG, "\tINLIN (inline data)");
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_VLAN)
		DRV_LOG(DEBUG, "\tVLANI (VLAN insertion)");
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_METADATA)
		DRV_LOG(DEBUG, "\tMETAD (tx Flow metadata)");
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_TXPP)
		DRV_LOG(DEBUG, "\tMETAD (tx Scheduling)");
	if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_EMPW) {
		if (txoff_func[m].olx & MLX5_TXOFF_CONFIG_MPW)
			DRV_LOG(DEBUG, "\tMPW   (Legacy MPW)");
		else
			DRV_LOG(DEBUG, "\tEMPW  (Enhanced MPW)");
	}
	return txoff_func[m].func;
}


mlx5_tx_packet_multi_tso(struct mlx5_txq_data *__rte_restrict txq,
			struct mlx5_txq_local *__rte_restrict loc,
			unsigned int olx)
{
	struct mlx5_wqe *__rte_restrict wqe;
	unsigned int ds, dlen, inlen, ntcp, vlan = 0;

	if (MLX5_TXOFF_CONFIG(TXPP)) {
		enum mlx5_txcmp_code wret;

		/* Generate WAIT for scheduling if requested. */
		wret = mlx5_tx_schedule_send(txq, loc, olx);
		if (wret == MLX5_TXCMP_CODE_EXIT)
			return MLX5_TXCMP_CODE_EXIT;
		if (wret == MLX5_TXCMP_CODE_ERROR)
			return MLX5_TXCMP_CODE_ERROR;
	}
	/*
	 * Calculate data length to be inlined to estimate
	 * the required space in WQE ring buffer.
	 */
	dlen = rte_pktmbuf_pkt_len(loc->mbuf);
	if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)
		vlan = sizeof(struct rte_vlan_hdr);
	inlen = loc->mbuf->l2_len + vlan +
		loc->mbuf->l3_len + loc->mbuf->l4_len;
	if (unlikely((!inlen || !loc->mbuf->tso_segsz)))
		return MLX5_TXCMP_CODE_ERROR;
	if (loc->mbuf->ol_flags & PKT_TX_TUNNEL_MASK)
		inlen += loc->mbuf->outer_l2_len + loc->mbuf->outer_l3_len;
	/* Packet must contain all TSO headers. */
	if (unlikely(inlen > MLX5_MAX_TSO_HEADER ||
		     inlen <= MLX5_ESEG_MIN_INLINE_SIZE ||
		     inlen > (dlen + vlan)))
		return MLX5_TXCMP_CODE_ERROR;
	/*
	 * Check whether there are enough free WQEBBs:
	 * - Control Segment
	 * - Ethernet Segment
	 * - First Segment of inlined Ethernet data
	 * - ... data continued ...
	 * - Data Segments of pointer/min inline type
	 */
	ds = NB_SEGS(loc->mbuf) + 2 + (inlen -
				       MLX5_ESEG_MIN_INLINE_SIZE +
				       MLX5_WSEG_SIZE +
				       MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
	if (unlikely(loc->wqe_free < ((ds + 3) / 4)))
		return MLX5_TXCMP_CODE_EXIT;
	/* Check for maximal WQE size. */
	if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4)))
		return MLX5_TXCMP_CODE_ERROR;
#ifdef MLX5_PMD_SOFT_COUNTERS
	/* Update sent data bytes/packets counters. */
	ntcp = (dlen - (inlen - vlan) + loc->mbuf->tso_segsz - 1) /
		loc->mbuf->tso_segsz;
	/*
	 * One will be added for mbuf itself
	 * at the end of the mlx5_tx_burst from
	 * loc->pkts_sent field.
	 */
	--ntcp;
	txq->stats.opackets += ntcp;
	txq->stats.obytes += dlen + vlan + ntcp * inlen;
#endif
	wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
	loc->wqe_last = wqe;
	mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_TSO, olx);
	ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 1, olx);
	wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);
	txq->wqe_ci += (ds + 3) / 4;
	loc->wqe_free -= (ds + 3) / 4;
	return MLX5_TXCMP_CODE_MULTI;
}


mlx5_tx_packet_multi_send(struct mlx5_txq_data *restrict txq,
			  struct mlx5_txq_local *restrict loc,
			  unsigned int olx)
{
	struct mlx5_wqe_dseg *restrict dseg;
	struct mlx5_wqe *restrict wqe;
	unsigned int ds, nseg;

	assert(NB_SEGS(loc->mbuf) > 1);
	/*
	 * No inline at all, it means the CPU cycles saving
	 * is prioritized at configuration, we should not
	 * copy any packet data to WQE.
	 */
	nseg = NB_SEGS(loc->mbuf);
	ds = 2 + nseg;
	if (unlikely(loc->wqe_free < ((ds + 3) / 4)))
		return MLX5_TXCMP_CODE_EXIT;
	/* Check for maximal WQE size. */
	if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ((ds + 3) / 4)))
		return MLX5_TXCMP_CODE_ERROR;
	/*
	 * Some Tx offloads may cause an error if
	 * packet is not long enough, check against
	 * assumed minimal length.
	 */
	if (rte_pktmbuf_pkt_len(loc->mbuf) <= MLX5_ESEG_MIN_INLINE_SIZE)
		return MLX5_TXCMP_CODE_ERROR;
#ifdef MLX5_PMD_SOFT_COUNTERS
	/* Update sent data bytes counter. */
	txq->stats.obytes += rte_pktmbuf_pkt_len(loc->mbuf);
	if (MLX5_TXOFF_CONFIG(VLAN) &&
	    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)
		txq->stats.obytes += sizeof(struct rte_vlan_hdr);
#endif
	/*
	 * SEND WQE, one WQEBB:
	 * - Control Segment, SEND opcode
	 * - Ethernet Segment, optional VLAN, no inline
	 * - Data Segments, pointer only type
	 */
	wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
	loc->wqe_last = wqe;
	mlx5_tx_cseg_init(txq, loc, wqe, ds, MLX5_OPCODE_SEND, olx);
	mlx5_tx_eseg_none(txq, loc, wqe, olx);
	dseg = &wqe->dseg[0];
	do {
		if (unlikely(!rte_pktmbuf_data_len(loc->mbuf))) {
			struct rte_mbuf *mbuf;

			/*
			 * Zero length segment found, have to
			 * correct total size of WQE in segments.
			 * It is supposed to be rare occasion, so
			 * in normal case (no zero length segments)
			 * we avoid extra writing to the Control
			 * Segment.
			 */
			--ds;
			wqe->cseg.sq_ds -= RTE_BE32(1);
			mbuf = loc->mbuf;
			loc->mbuf = mbuf->next;
			rte_pktmbuf_free_seg(mbuf);
			if (--nseg == 0)
				break;
		} else {
			mlx5_tx_dseg_ptr
				(txq, loc, dseg,
				 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
				 rte_pktmbuf_data_len(loc->mbuf), olx);
			txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
			--loc->elts_free;
			if (--nseg == 0)
				break;
			++dseg;
			if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
				dseg = (struct mlx5_wqe_dseg *)txq->wqes;
			loc->mbuf = loc->mbuf->next;
		}
	} while (true);
	txq->wqe_ci += (ds + 3) / 4;
	loc->wqe_free -= (ds + 3) / 4;
	return MLX5_TXCMP_CODE_MULTI;
}


mlx5_tx_burst_empw_inline(struct mlx5_txq_data *restrict txq,
			  struct rte_mbuf **restrict pkts,
			  unsigned int pkts_n,
			  struct mlx5_txq_local *restrict loc,
			  unsigned int olx)
{
	/*
	 * Subroutine is the part of mlx5_tx_burst_single()
	 * and sends single-segment packet with eMPW opcode
	 * with data inlining.
	 */
	assert(MLX5_TXOFF_CONFIG(INLINE));
	assert(MLX5_TXOFF_CONFIG(EMPW));
	assert(loc->elts_free && loc->wqe_free);
	assert(pkts_n > loc->pkts_sent);
	static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
	pkts += loc->pkts_sent + 1;
	pkts_n -= loc->pkts_sent;
	for (;;) {
		struct mlx5_wqe_dseg *restrict dseg;
		struct mlx5_wqe_eseg *restrict eseg;
		enum mlx5_txcmp_code ret;
		unsigned int room, part, nlim;
		unsigned int slen = 0;

		assert(NB_SEGS(loc->mbuf) == 1);
		/*
		 * Limits the amount of packets in one WQE
		 * to improve CQE latency generation.
		 */
		nlim = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ?
				       MLX5_MPW_INLINE_MAX_PACKETS :
				       MLX5_EMPW_MAX_PACKETS);
		/* Check whether we have minimal amount WQEs */
		if (unlikely(loc->wqe_free <
			    ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4)))
			return MLX5_TXCMP_CODE_EXIT;
		if (likely(pkts_n > 1))
			rte_prefetch0(*pkts);
		loc->wqe_last = txq->wqes + (txq->wqe_ci & txq->wqe_m);
		/*
		 * Build eMPW title WQEBB:
		 * - Control Segment, eMPW opcode, zero DS
		 * - Ethernet Segment, no inline
		 */
		mlx5_tx_cseg_init(txq, loc, loc->wqe_last, 0,
				  MLX5_OPCODE_ENHANCED_MPSW, olx);
		mlx5_tx_eseg_none(txq, loc, loc->wqe_last,
				  olx & ~MLX5_TXOFF_CONFIG_VLAN);
		eseg = &loc->wqe_last->eseg;
		dseg = &loc->wqe_last->dseg[0];
		/* Store the packet length for legacy MPW. */
		if (MLX5_TXOFF_CONFIG(MPW))
			eseg->mss = rte_cpu_to_be_16
					(rte_pktmbuf_data_len(loc->mbuf));
		room = RTE_MIN(MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE,
			       loc->wqe_free) * MLX5_WQE_SIZE -
					MLX5_WQE_CSEG_SIZE -
					MLX5_WQE_ESEG_SIZE;
		/* Limit the room for legacy MPW sessions for performance. */
		if (MLX5_TXOFF_CONFIG(MPW))
			room = RTE_MIN(room,
				       RTE_MAX(txq->inlen_empw +
					       sizeof(dseg->bcount) +
					       (MLX5_TXOFF_CONFIG(VLAN) ?
					       sizeof(struct rte_vlan_hdr) : 0),
					       MLX5_MPW_INLINE_MAX_PACKETS *
					       MLX5_WQE_DSEG_SIZE));
		/* Build WQE till we have space, packets and resources. */
		part = room;
		for (;;) {
			uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf);
			uint8_t *dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);
			unsigned int tlen;

			assert(room >= MLX5_WQE_DSEG_SIZE);
			assert((room % MLX5_WQE_DSEG_SIZE) == 0);
			assert((uintptr_t)dseg < (uintptr_t)txq->wqes_end);
			/*
			 * Some Tx offloads may cause an error if
			 * packet is not long enough, check against
			 * assumed minimal length.
			 */
			if (unlikely(dlen <= MLX5_ESEG_MIN_INLINE_SIZE)) {
				part -= room;
				if (unlikely(!part))
					return MLX5_TXCMP_CODE_ERROR;
				/*
				 * We have some successfully built
				 * packet Data Segments to send.
				 */
				mlx5_tx_idone_empw(txq, loc, part, slen, olx);
				return MLX5_TXCMP_CODE_ERROR;
			}
			/* Inline or not inline - that's the Question. */
			if (dlen > txq->inlen_empw)
				goto pointer_empw;
			if (MLX5_TXOFF_CONFIG(MPW)) {
				if (dlen > txq->inlen_send)
					goto pointer_empw;
				tlen = dlen;
				if (part == room) {
					/* Open new inline MPW session. */
					tlen += sizeof(dseg->bcount);
					dseg->bcount = RTE_BE32(0);
					dseg = RTE_PTR_ADD
						(dseg, sizeof(dseg->bcount));
				} else {
					/*
					 * No pointer and inline descriptor
					 * intermix for legacy MPW sessions.
					 */
					if (loc->wqe_last->dseg[0].bcount)
						break;
				}
			} else {
				tlen = sizeof(dseg->bcount) + dlen;
			}
			/* Inline entire packet, optional VLAN insertion. */
			if (MLX5_TXOFF_CONFIG(VLAN) &&
			    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {
				/*
				 * The packet length must be checked in
				 * mlx5_tx_able_to_empw() and packet
				 * fits into inline length guaranteed.
				 */
				assert((dlen + sizeof(struct rte_vlan_hdr)) <=
					txq->inlen_empw);
				tlen += sizeof(struct rte_vlan_hdr);
				if (room < tlen)
					break;
				dseg = mlx5_tx_dseg_vlan(txq, loc, dseg,
							 dptr, dlen, olx);
#ifdef MLX5_PMD_SOFT_COUNTERS
				/* Update sent data bytes counter. */
				slen +=	sizeof(struct rte_vlan_hdr);
#endif
			} else {
				if (room < tlen)
					break;
				dseg = mlx5_tx_dseg_empw(txq, loc, dseg,
							 dptr, dlen, olx);
			}
			if (!MLX5_TXOFF_CONFIG(MPW))
				tlen = RTE_ALIGN(tlen, MLX5_WSEG_SIZE);
			assert(room >= tlen);
			room -= tlen;
			/*
			 * Packet data are completely inlined,
			 * free the packet immediately.
			 */
			rte_pktmbuf_free_seg(loc->mbuf);
			goto next_mbuf;
pointer_empw:
			/*
			 * No pointer and inline descriptor
			 * intermix for legacy MPW sessions.
			 */
			if (MLX5_TXOFF_CONFIG(MPW) &&
			    part != room &&
			    loc->wqe_last->dseg[0].bcount == RTE_BE32(0))
				break;
			/*
			 * Not inlinable VLAN packets are
			 * proceeded outside of this routine.
			 */
			assert(room >= MLX5_WQE_DSEG_SIZE);
			if (MLX5_TXOFF_CONFIG(VLAN))
				assert(!(loc->mbuf->ol_flags &
					 PKT_TX_VLAN_PKT));
			mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);
			/* We have to store mbuf in elts.*/
			txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
			room -= MLX5_WQE_DSEG_SIZE;
			/* Ring buffer wraparound is checked at the loop end.*/
			++dseg;
next_mbuf:
#ifdef MLX5_PMD_SOFT_COUNTERS
			/* Update sent data bytes counter. */
			slen += dlen;
#endif
			loc->pkts_sent++;
			loc->elts_free--;
			pkts_n--;
			if (unlikely(!pkts_n || !loc->elts_free)) {
				/*
				 * We have no resources/packets to
				 * continue build descriptors.
				 */
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part, slen, olx);
				return MLX5_TXCMP_CODE_EXIT;
			}
			loc->mbuf = *pkts++;
			if (likely(pkts_n > 1))
				rte_prefetch0(*pkts);
			ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
			/*
			 * Unroll the completion code to avoid
			 * returning variable value - it results in
			 * unoptimized sequent checking in caller.
			 */
			if (ret == MLX5_TXCMP_CODE_MULTI) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part, slen, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_MULTI;
			}
			assert(NB_SEGS(loc->mbuf) == 1);
			if (ret == MLX5_TXCMP_CODE_TSO) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part, slen, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_TSO;
			}
			if (ret == MLX5_TXCMP_CODE_SINGLE) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part, slen, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_SINGLE;
			}
			if (ret != MLX5_TXCMP_CODE_EMPW) {
				assert(false);
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part, slen, olx);
				return MLX5_TXCMP_CODE_ERROR;
			}
			/* Check if we have minimal room left. */
			nlim--;
			if (unlikely(!nlim || room < MLX5_WQE_DSEG_SIZE))
				break;
			/*
			 * Check whether packet parameters coincide
			 * within assumed eMPW batch:
			 * - check sum settings
			 * - metadata value
			 * - software parser settings
			 * - packets length (legacy MPW only)
			 */
			if (!mlx5_tx_match_empw(txq, eseg, loc, dlen, olx))
				break;
			/* Packet attributes match, continue the same eMPW. */
			if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
				dseg = (struct mlx5_wqe_dseg *)txq->wqes;
		}
		/*
		 * We get here to close an existing eMPW
		 * session and start the new one.
		 */
		assert(pkts_n);
		part -= room;
		if (unlikely(!part))
			return MLX5_TXCMP_CODE_EXIT;
		mlx5_tx_idone_empw(txq, loc, part, slen, olx);
		if (unlikely(!loc->elts_free ||
			     !loc->wqe_free))
			return MLX5_TXCMP_CODE_EXIT;
		/* Continue the loop with new eMPW session. */
	}
	assert(false);
}


mlx5_tx_eseg_none(struct mlx5_txq_data *restrict txq __rte_unused,
		  struct mlx5_txq_local *restrict loc,
		  struct mlx5_wqe *restrict wqe,
		  unsigned int olx)
{
	struct mlx5_wqe_eseg *restrict es = &wqe->eseg;
	uint32_t csum;

	/*
	 * Calculate and set check sum flags first, dword field
	 * in segment may be shared with Software Parser flags.
	 */
	csum = MLX5_TXOFF_CONFIG(CSUM) ? txq_ol_cksum_to_cs(loc->mbuf) : 0;
	es->flags = rte_cpu_to_le_32(csum);
	/*
	 * Calculate and set Software Parser offsets and flags.
	 * These flags a set for custom UDP and IP tunnel packets.
	 */
	es->swp_offs = txq_mbuf_to_swp(loc, &es->swp_flags, olx);
	/* Fill metadata field if needed. */
	es->metadata = MLX5_TXOFF_CONFIG(METADATA) ?
		       loc->mbuf->ol_flags & PKT_TX_DYNF_METADATA ?
		       *RTE_FLOW_DYNF_METADATA(loc->mbuf) : 0 : 0;
	/* Engage VLAN tag insertion feature if requested. */
	if (MLX5_TXOFF_CONFIG(VLAN) &&
	    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {
		/*
		 * We should get here only if device support
		 * this feature correctly.
		 */
		assert(txq->vlan_en);
		es->inline_hdr = rte_cpu_to_be_32(MLX5_ETH_WQE_VLAN_INSERT |
						  loc->mbuf->vlan_tci);
	} else {
		es->inline_hdr = RTE_BE32(0);
	}
}


mlx5_tx_able_to_empw(struct mlx5_txq_data *restrict txq,
		     struct mlx5_txq_local *restrict loc,
		     unsigned int olx,
		     bool newp)
{
	/* Check for multi-segment packet. */
	if (newp &&
	    MLX5_TXOFF_CONFIG(MULTI) &&
	    unlikely(NB_SEGS(loc->mbuf) > 1))
		return MLX5_TXCMP_CODE_MULTI;
	/* Check for TSO packet. */
	if (newp &&
	    MLX5_TXOFF_CONFIG(TSO) &&
	    unlikely(loc->mbuf->ol_flags & PKT_TX_TCP_SEG))
		return MLX5_TXCMP_CODE_TSO;
	/* Check if eMPW is enabled at all. */
	if (!MLX5_TXOFF_CONFIG(EMPW))
		return MLX5_TXCMP_CODE_SINGLE;
	/* Check if eMPW can be engaged. */
	if (MLX5_TXOFF_CONFIG(VLAN) &&
	    unlikely(loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) &&
		(!MLX5_TXOFF_CONFIG(INLINE) ||
		 unlikely((rte_pktmbuf_data_len(loc->mbuf) +
			   sizeof(struct rte_vlan_hdr)) > txq->inlen_empw))) {
		/*
		 * eMPW does not support VLAN insertion offload,
		 * we have to inline the entire packet but
		 * packet is too long for inlining.
		 */
		return MLX5_TXCMP_CODE_SINGLE;
	}
	return MLX5_TXCMP_CODE_EMPW;
}


mlx5_tx_burst_tso(struct mlx5_txq_data *restrict txq,
		  struct rte_mbuf **restrict pkts,
		  unsigned int pkts_n,
		  struct mlx5_txq_local *restrict loc,
		  unsigned int olx)
{
	assert(loc->elts_free && loc->wqe_free);
	assert(pkts_n > loc->pkts_sent);
	pkts += loc->pkts_sent + 1;
	pkts_n -= loc->pkts_sent;
	for (;;) {
		struct mlx5_wqe_dseg *restrict dseg;
		struct mlx5_wqe *restrict wqe;
		unsigned int ds, dlen, hlen, ntcp, vlan = 0;
		uint8_t *dptr;

		assert(NB_SEGS(loc->mbuf) == 1);
		dlen = rte_pktmbuf_data_len(loc->mbuf);
		if (MLX5_TXOFF_CONFIG(VLAN) &&
		    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {
			vlan = sizeof(struct rte_vlan_hdr);
		}
		/*
		 * First calculate the WQE size to check
		 * whether we have enough space in ring buffer.
		 */
		hlen = loc->mbuf->l2_len + vlan +
		       loc->mbuf->l3_len + loc->mbuf->l4_len;
		if (unlikely((!hlen || !loc->mbuf->tso_segsz)))
			return MLX5_TXCMP_CODE_ERROR;
		if (loc->mbuf->ol_flags & PKT_TX_TUNNEL_MASK)
			hlen += loc->mbuf->outer_l2_len +
				loc->mbuf->outer_l3_len;
		/* Segment must contain all TSO headers. */
		if (unlikely(hlen > MLX5_MAX_TSO_HEADER ||
			     hlen <= MLX5_ESEG_MIN_INLINE_SIZE ||
			     hlen > (dlen + vlan)))
			return MLX5_TXCMP_CODE_ERROR;
		/*
		 * Check whether there are enough free WQEBBs:
		 * - Control Segment
		 * - Ethernet Segment
		 * - First Segment of inlined Ethernet data
		 * - ... data continued ...
		 * - Finishing Data Segment of pointer type
		 */
		ds = 4 + (hlen - MLX5_ESEG_MIN_INLINE_SIZE +
			  MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
		if (loc->wqe_free < ((ds + 3) / 4))
			return MLX5_TXCMP_CODE_EXIT;
#ifdef MLX5_PMD_SOFT_COUNTERS
		/* Update sent data bytes/packets counters. */
		ntcp = (dlen + vlan - hlen +
			loc->mbuf->tso_segsz - 1) /
			loc->mbuf->tso_segsz;
		/*
		 * One will be added for mbuf itself at the end
		 * of the mlx5_tx_burst from loc->pkts_sent field.
		 */
		--ntcp;
		txq->stats.opackets += ntcp;
		txq->stats.obytes += dlen + vlan + ntcp * hlen;
#endif
		/*
		 * Build the TSO WQE:
		 * - Control Segment
		 * - Ethernet Segment with hlen bytes inlined
		 * - Data Segment of pointer type
		 */
		wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
		loc->wqe_last = wqe;
		mlx5_tx_cseg_init(txq, loc, wqe, ds,
				  MLX5_OPCODE_TSO, olx);
		dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan, hlen, 1, olx);
		dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) + hlen - vlan;
		dlen -= hlen - vlan;
		mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);
		/*
		 * WQE is built, update the loop parameters
		 * and go to the next packet.
		 */
		txq->wqe_ci += (ds + 3) / 4;
		loc->wqe_free -= (ds + 3) / 4;
		if (MLX5_TXOFF_CONFIG(INLINE))
			txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
		--loc->elts_free;
		++loc->pkts_sent;
		--pkts_n;
		if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))
			return MLX5_TXCMP_CODE_EXIT;
		loc->mbuf = *pkts++;
		if (pkts_n > 1)
			rte_prefetch0(*pkts);
		if (MLX5_TXOFF_CONFIG(MULTI) &&
		    unlikely(NB_SEGS(loc->mbuf) > 1))
			return MLX5_TXCMP_CODE_MULTI;
		if (likely(!(loc->mbuf->ol_flags & PKT_TX_TCP_SEG)))
			return MLX5_TXCMP_CODE_SINGLE;
		/* Continue with the next TSO packet. */
	}
	assert(false);
}


static int packet_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
			  int flags)
{
	struct sock *sk = sock->sk;
	struct sk_buff *skb;
	int copied, err;
	int vnet_hdr_len = 0;
	unsigned int origlen = 0;

	err = -EINVAL;
	if (flags & ~(MSG_PEEK|MSG_DONTWAIT|MSG_TRUNC|MSG_CMSG_COMPAT|MSG_ERRQUEUE))
		goto out;

#if 0
	/* What error should we return now? EUNATTACH? */
	if (pkt_sk(sk)->ifindex < 0)
		return -ENODEV;
#endif

	if (flags & MSG_ERRQUEUE) {
		err = sock_recv_errqueue(sk, msg, len,
					 SOL_PACKET, PACKET_TX_TIMESTAMP);
		goto out;
	}

	/*
	 *	Call the generic datagram receiver. This handles all sorts
	 *	of horrible races and re-entrancy so we can forget about it
	 *	in the protocol layers.
	 *
	 *	Now it will return ENETDOWN, if device have just gone down,
	 *	but then it will block.
	 */

	skb = skb_recv_datagram(sk, flags, flags & MSG_DONTWAIT, &err);

	/*
	 *	An error occurred so return it. Because skb_recv_datagram()
	 *	handles the blocking we don't see and worry about blocking
	 *	retries.
	 */

	if (skb == NULL)
		goto out;

	packet_rcv_try_clear_pressure(pkt_sk(sk));

	if (pkt_sk(sk)->has_vnet_hdr) {
		err = packet_rcv_vnet(msg, skb, &len);
		if (err)
			goto out_free;
		vnet_hdr_len = sizeof(struct virtio_net_hdr);
	}

	/* You lose any data beyond the buffer you gave. If it worries
	 * a user program they can ask the device for its MTU
	 * anyway.
	 */
	copied = skb->len;
	if (copied > len) {
		copied = len;
		msg->msg_flags |= MSG_TRUNC;
	}

	err = skb_copy_datagram_msg(skb, 0, msg, copied);
	if (err)
		goto out_free;

	if (sock->type != SOCK_PACKET) {
		struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;

		/* Original length was stored in sockaddr_ll fields */
		origlen = PACKET_SKB_CB(skb)->sa.origlen;
		sll->sll_family = AF_PACKET;
		sll->sll_protocol = skb->protocol;
	}

	sock_recv_ts_and_drops(msg, sk, skb);

	if (msg->msg_name) {
		const size_t max_len = min(sizeof(skb->cb),
					   sizeof(struct sockaddr_storage));
		int copy_len;

		/* If the address length field is there to be filled
		 * in, we fill it in now.
		 */
		if (sock->type == SOCK_PACKET) {
			__sockaddr_check_size(sizeof(struct sockaddr_pkt));
			msg->msg_namelen = sizeof(struct sockaddr_pkt);
			copy_len = msg->msg_namelen;
		} else {
			struct sockaddr_ll *sll = &PACKET_SKB_CB(skb)->sa.ll;

			msg->msg_namelen = sll->sll_halen +
				offsetof(struct sockaddr_ll, sll_addr);
			copy_len = msg->msg_namelen;
			if (msg->msg_namelen < sizeof(struct sockaddr_ll)) {
				memset(msg->msg_name +
				       offsetof(struct sockaddr_ll, sll_addr),
				       0, sizeof(sll->sll_addr));
				msg->msg_namelen = sizeof(struct sockaddr_ll);
			}
		}
		if (WARN_ON_ONCE(copy_len > max_len)) {
			copy_len = max_len;
			msg->msg_namelen = copy_len;
		}
		memcpy(msg->msg_name, &PACKET_SKB_CB(skb)->sa, copy_len);
	}

	if (pkt_sk(sk)->auxdata) {
		struct tpacket_auxdata aux;

		aux.tp_status = TP_STATUS_USER;
		if (skb->ip_summed == CHECKSUM_PARTIAL)
			aux.tp_status |= TP_STATUS_CSUMNOTREADY;
		else if (skb->pkt_type != PACKET_OUTGOING &&
			 (skb->ip_summed == CHECKSUM_COMPLETE ||
			  skb_csum_unnecessary(skb)))
			aux.tp_status |= TP_STATUS_CSUM_VALID;

		aux.tp_len = origlen;
		aux.tp_snaplen = skb->len;
		aux.tp_mac = 0;
		aux.tp_net = skb_network_offset(skb);
		if (skb_vlan_tag_present(skb)) {
			aux.tp_vlan_tci = skb_vlan_tag_get(skb);
			aux.tp_vlan_tpid = ntohs(skb->vlan_proto);
			aux.tp_status |= TP_STATUS_VLAN_VALID | TP_STATUS_VLAN_TPID_VALID;
		} else {
			aux.tp_vlan_tci = 0;
			aux.tp_vlan_tpid = 0;
		}
		put_cmsg(msg, SOL_PACKET, PACKET_AUXDATA, sizeof(aux), &aux);
	}

	/*
	 *	Free or return the buffer as appropriate. Again this
	 *	hides all the races and re-entrancy issues from us.
	 */
	err = vnet_hdr_len + ((flags&MSG_TRUNC) ? skb->len : copied);

out_free:
	skb_free_datagram(sk, skb);
out:
	return err;
}


static int packet_notifier(struct notifier_block *this,
			   unsigned long msg, void *ptr)
{
	struct sock *sk;
	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
	struct net *net = dev_net(dev);

	rcu_read_lock();
	sk_for_each_rcu(sk, &net->packet.sklist) {
		struct packet_sock *po = pkt_sk(sk);

		switch (msg) {
		case NETDEV_UNREGISTER:
			if (po->mclist)
				packet_dev_mclist_delete(dev, &po->mclist);
			fallthrough;

		case NETDEV_DOWN:
			if (dev->ifindex == po->ifindex) {
				spin_lock(&po->bind_lock);
				if (po->running) {
					__unregister_prot_hook(sk, false);
					sk->sk_err = ENETDOWN;
					if (!sock_flag(sk, SOCK_DEAD))
						sk_error_report(sk);
				}
				if (msg == NETDEV_UNREGISTER) {
					packet_cached_dev_reset(po);
					WRITE_ONCE(po->ifindex, -1);
					dev_put_track(po->prot_hook.dev,
						      &po->prot_hook.dev_tracker);
					po->prot_hook.dev = NULL;
				}
				spin_unlock(&po->bind_lock);
			}
			break;
		case NETDEV_UP:
			if (dev->ifindex == po->ifindex) {
				spin_lock(&po->bind_lock);
				if (po->num)
					register_prot_hook(sk);
				spin_unlock(&po->bind_lock);
			}
			break;
		}
	}
	rcu_read_unlock();
	return NOTIFY_DONE;
}


do_mouse(
    oparg_T	*oap,		// operator argument, can be NULL
    int		c,		// K_LEFTMOUSE, etc
    int		dir,		// Direction to 'put' if necessary
    long	count,
    int		fixindent)	// PUT_FIXINDENT if fixing indent necessary
{
    static int	do_always = FALSE;	// ignore 'mouse' setting next time
    static int	got_click = FALSE;	// got a click some time back

    int		which_button;	// MOUSE_LEFT, _MIDDLE or _RIGHT
    int		is_click = FALSE; // If FALSE it's a drag or release event
    int		is_drag = FALSE;  // If TRUE it's a drag event
    int		jump_flags = 0;	// flags for jump_to_mouse()
    pos_T	start_visual;
    int		moved;		// Has cursor moved?
    int		in_status_line;	// mouse in status line
    static int	in_tab_line = FALSE; // mouse clicked in tab line
    int		in_sep_line;	// mouse in vertical separator line
    int		c1, c2;
#if defined(FEAT_FOLDING)
    pos_T	save_cursor;
#endif
    win_T	*old_curwin = curwin;
    static pos_T orig_cursor;
    colnr_T	leftcol, rightcol;
    pos_T	end_visual;
    int		diff;
    int		old_active = VIsual_active;
    int		old_mode = VIsual_mode;
    int		regname;

#if defined(FEAT_FOLDING)
    save_cursor = curwin->w_cursor;
#endif

    // When GUI is active, always recognize mouse events, otherwise:
    // - Ignore mouse event in normal mode if 'mouse' doesn't include 'n'.
    // - Ignore mouse event in visual mode if 'mouse' doesn't include 'v'.
    // - For command line and insert mode 'mouse' is checked before calling
    //	 do_mouse().
    if (do_always)
	do_always = FALSE;
    else
#ifdef FEAT_GUI
	if (!gui.in_use)
#endif
	{
	    if (VIsual_active)
	    {
		if (!mouse_has(MOUSE_VISUAL))
		    return FALSE;
	    }
	    else if (State == MODE_NORMAL && !mouse_has(MOUSE_NORMAL))
		return FALSE;
	}

    for (;;)
    {
	which_button = get_mouse_button(KEY2TERMCAP1(c), &is_click, &is_drag);
	if (is_drag)
	{
	    // If the next character is the same mouse event then use that
	    // one. Speeds up dragging the status line.
	    // Note: Since characters added to the stuff buffer in the code
	    // below need to come before the next character, do not do this
	    // when the current character was stuffed.
	    if (!KeyStuffed && vpeekc() != NUL)
	    {
		int nc;
		int save_mouse_row = mouse_row;
		int save_mouse_col = mouse_col;

		// Need to get the character, peeking doesn't get the actual
		// one.
		nc = safe_vgetc();
		if (c == nc)
		    continue;
		vungetc(nc);
		mouse_row = save_mouse_row;
		mouse_col = save_mouse_col;
	    }
	}
	break;
    }

    if (c == K_MOUSEMOVE)
    {
	// Mouse moved without a button pressed.
#ifdef FEAT_BEVAL_TERM
	ui_may_remove_balloon();
	if (p_bevalterm)
	{
	    profile_setlimit(p_bdlay, &bevalexpr_due);
	    bevalexpr_due_set = TRUE;
	}
#endif
#ifdef FEAT_PROP_POPUP
	popup_handle_mouse_moved();
#endif
	return FALSE;
    }

#ifdef FEAT_MOUSESHAPE
    // May have stopped dragging the status or separator line.  The pointer is
    // most likely still on the status or separator line.
    if (!is_drag && drag_status_line)
    {
	drag_status_line = FALSE;
	update_mouseshape(SHAPE_IDX_STATUS);
    }
    if (!is_drag && drag_sep_line)
    {
	drag_sep_line = FALSE;
	update_mouseshape(SHAPE_IDX_VSEP);
    }
#endif

    // Ignore drag and release events if we didn't get a click.
    if (is_click)
	got_click = TRUE;
    else
    {
	if (!got_click)			// didn't get click, ignore
	    return FALSE;
	if (!is_drag)			// release, reset got_click
	{
	    got_click = FALSE;
	    if (in_tab_line)
	    {
		in_tab_line = FALSE;
		return FALSE;
	    }
	}
    }

    // CTRL right mouse button does CTRL-T
    if (is_click && (mod_mask & MOD_MASK_CTRL) && which_button == MOUSE_RIGHT)
    {
	if (State & MODE_INSERT)
	    stuffcharReadbuff(Ctrl_O);
	if (count > 1)
	    stuffnumReadbuff(count);
	stuffcharReadbuff(Ctrl_T);
	got_click = FALSE;		// ignore drag&release now
	return FALSE;
    }

    // CTRL only works with left mouse button
    if ((mod_mask & MOD_MASK_CTRL) && which_button != MOUSE_LEFT)
	return FALSE;

    // When a modifier is down, ignore drag and release events, as well as
    // multiple clicks and the middle mouse button.
    // Accept shift-leftmouse drags when 'mousemodel' is "popup.*".
    if ((mod_mask & (MOD_MASK_SHIFT | MOD_MASK_CTRL | MOD_MASK_ALT
							     | MOD_MASK_META))
	    && (!is_click
		|| (mod_mask & MOD_MASK_MULTI_CLICK)
		|| which_button == MOUSE_MIDDLE)
	    && !((mod_mask & (MOD_MASK_SHIFT|MOD_MASK_ALT))
		&& mouse_model_popup()
		&& which_button == MOUSE_LEFT)
	    && !((mod_mask & MOD_MASK_ALT)
		&& !mouse_model_popup()
		&& which_button == MOUSE_RIGHT)
	    )
	return FALSE;

    // If the button press was used as the movement command for an operator
    // (eg "d<MOUSE>"), or it is the middle button that is held down, ignore
    // drag/release events.
    if (!is_click && which_button == MOUSE_MIDDLE)
	return FALSE;

    if (oap != NULL)
	regname = oap->regname;
    else
	regname = 0;

    // Middle mouse button does a 'put' of the selected text
    if (which_button == MOUSE_MIDDLE)
    {
	if (State == MODE_NORMAL)
	{
	    // If an operator was pending, we don't know what the user wanted
	    // to do. Go back to normal mode: Clear the operator and beep().
	    if (oap != NULL && oap->op_type != OP_NOP)
	    {
		clearopbeep(oap);
		return FALSE;
	    }

	    // If visual was active, yank the highlighted text and put it
	    // before the mouse pointer position.
	    // In Select mode replace the highlighted text with the clipboard.
	    if (VIsual_active)
	    {
		if (VIsual_select)
		{
		    stuffcharReadbuff(Ctrl_G);
		    stuffReadbuff((char_u *)"\"+p");
		}
		else
		{
		    stuffcharReadbuff('y');
		    stuffcharReadbuff(K_MIDDLEMOUSE);
		}
		do_always = TRUE;	// ignore 'mouse' setting next time
		return FALSE;
	    }
	    // The rest is below jump_to_mouse()
	}

	else if ((State & MODE_INSERT) == 0)
	    return FALSE;

	// Middle click in insert mode doesn't move the mouse, just insert the
	// contents of a register.  '.' register is special, can't insert that
	// with do_put().
	// Also paste at the cursor if the current mode isn't in 'mouse' (only
	// happens for the GUI).
	if ((State & MODE_INSERT) || !mouse_has(MOUSE_NORMAL))
	{
	    if (regname == '.')
		insert_reg(regname, TRUE);
	    else
	    {
#ifdef FEAT_CLIPBOARD
		if (clip_star.available && regname == 0)
		    regname = '*';
#endif
		if ((State & REPLACE_FLAG) && !yank_register_mline(regname))
		    insert_reg(regname, TRUE);
		else
		{
		    do_put(regname, NULL, BACKWARD, 1L,
						      fixindent | PUT_CURSEND);

		    // Repeat it with CTRL-R CTRL-O r or CTRL-R CTRL-P r
		    AppendCharToRedobuff(Ctrl_R);
		    AppendCharToRedobuff(fixindent ? Ctrl_P : Ctrl_O);
		    AppendCharToRedobuff(regname == 0 ? '"' : regname);
		}
	    }
	    return FALSE;
	}
    }

    // When dragging or button-up stay in the same window.
    if (!is_click)
	jump_flags |= MOUSE_FOCUS | MOUSE_DID_MOVE;

    start_visual.lnum = 0;

    if (TabPageIdxs != NULL)  // only when initialized
    {
	// Check for clicking in the tab page line.
	if (mouse_row == 0 && firstwin->w_winrow > 0)
	{
	    if (is_drag)
	    {
		if (in_tab_line)
		{
		    c1 = TabPageIdxs[mouse_col];
		    tabpage_move(c1 <= 0 ? 9999 : c1 < tabpage_index(curtab)
								    ? c1 - 1 : c1);
		}
		return FALSE;
	    }

	    // click in a tab selects that tab page
	    if (is_click
# ifdef FEAT_CMDWIN
		    && cmdwin_type == 0
# endif
		    && mouse_col < Columns)
	    {
		in_tab_line = TRUE;
		c1 = TabPageIdxs[mouse_col];
		if (c1 >= 0)
		{
		    if ((mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_2CLICK)
		    {
			// double click opens new page
			end_visual_mode_keep_button();
			tabpage_new();
			tabpage_move(c1 == 0 ? 9999 : c1 - 1);
		    }
		    else
		    {
			// Go to specified tab page, or next one if not clicking
			// on a label.
			goto_tabpage(c1);

			// It's like clicking on the status line of a window.
			if (curwin != old_curwin)
			    end_visual_mode_keep_button();
		    }
		}
		else
		{
		    tabpage_T	*tp;

		    // Close the current or specified tab page.
		    if (c1 == -999)
			tp = curtab;
		    else
			tp = find_tabpage(-c1);
		    if (tp == curtab)
		    {
			if (first_tabpage->tp_next != NULL)
			    tabpage_close(FALSE);
		    }
		    else if (tp != NULL)
			tabpage_close_other(tp, FALSE);
		}
	    }
	    return TRUE;
	}
	else if (is_drag && in_tab_line)
	{
	    c1 = TabPageIdxs[mouse_col];
	    tabpage_move(c1 <= 0 ? 9999 : c1 - 1);
	    return FALSE;
	}
    }

    // When 'mousemodel' is "popup" or "popup_setpos", translate mouse events:
    // right button up   -> pop-up menu
    // shift-left button -> right button
    // alt-left button   -> alt-right button
    if (mouse_model_popup())
    {
	if (which_button == MOUSE_RIGHT
			    && !(mod_mask & (MOD_MASK_SHIFT | MOD_MASK_CTRL)))
	{
#ifdef USE_POPUP_SETPOS
# ifdef FEAT_GUI
	    if (gui.in_use)
	    {
#  if defined(FEAT_GUI_MOTIF) || defined(FEAT_GUI_GTK) \
			  || defined(FEAT_GUI_PHOTON)
		if (!is_click)
		    // Ignore right button release events, only shows the popup
		    // menu on the button down event.
		    return FALSE;
#  endif
#  if defined(FEAT_GUI_MSWIN) || defined(FEAT_GUI_HAIKU)
		if (is_click || is_drag)
		    // Ignore right button down and drag mouse events.  Windows
		    // only shows the popup menu on the button up event.
		    return FALSE;
#  endif
	    }
# endif
# if defined(FEAT_GUI) && defined(FEAT_TERM_POPUP_MENU)
	    else
# endif
# if defined(FEAT_TERM_POPUP_MENU)
	    if (!is_click)
		// Ignore right button release events, only shows the popup
		// menu on the button down event.
		return FALSE;
#endif

	    jump_flags = 0;
	    if (STRCMP(p_mousem, "popup_setpos") == 0)
	    {
		// First set the cursor position before showing the popup
		// menu.
		if (VIsual_active)
		{
		    pos_T    m_pos;

		    // set MOUSE_MAY_STOP_VIS if we are outside the
		    // selection or the current window (might have false
		    // negative here)
		    if (mouse_row < curwin->w_winrow
			 || mouse_row
				  > (curwin->w_winrow + curwin->w_height))
			jump_flags = MOUSE_MAY_STOP_VIS;
		    else if (get_fpos_of_mouse(&m_pos) != IN_BUFFER)
			jump_flags = MOUSE_MAY_STOP_VIS;
		    else
		    {
			if ((LT_POS(curwin->w_cursor, VIsual)
				    && (LT_POS(m_pos, curwin->w_cursor)
					|| LT_POS(VIsual, m_pos)))
				|| (LT_POS(VIsual, curwin->w_cursor)
				    && (LT_POS(m_pos, VIsual)
				      || LT_POS(curwin->w_cursor, m_pos))))
			{
			    jump_flags = MOUSE_MAY_STOP_VIS;
			}
			else if (VIsual_mode == Ctrl_V)
			{
			    getvcols(curwin, &curwin->w_cursor, &VIsual,
						     &leftcol, &rightcol);
			    getvcol(curwin, &m_pos, NULL, &m_pos.col, NULL);
			    if (m_pos.col < leftcol || m_pos.col > rightcol)
				jump_flags = MOUSE_MAY_STOP_VIS;
			}
		    }
		}
		else
		    jump_flags = MOUSE_MAY_STOP_VIS;
	    }
	    if (jump_flags)
	    {
		jump_flags = jump_to_mouse(jump_flags, NULL, which_button);
		update_curbuf(VIsual_active ? UPD_INVERTED : UPD_VALID);
		setcursor();
		out_flush();    // Update before showing popup menu
	    }
# ifdef FEAT_MENU
	    show_popupmenu();
	    got_click = FALSE;	// ignore release events
# endif
	    return (jump_flags & CURSOR_MOVED) != 0;
#else
	    return FALSE;
#endif
	}
	if (which_button == MOUSE_LEFT
				&& (mod_mask & (MOD_MASK_SHIFT|MOD_MASK_ALT)))
	{
	    which_button = MOUSE_RIGHT;
	    mod_mask &= ~MOD_MASK_SHIFT;
	}
    }

    if ((State & (MODE_NORMAL | MODE_INSERT))
			    && !(mod_mask & (MOD_MASK_SHIFT | MOD_MASK_CTRL)))
    {
	if (which_button == MOUSE_LEFT)
	{
	    if (is_click)
	    {
		// stop Visual mode for a left click in a window, but not when
		// on a status line
		if (VIsual_active)
		    jump_flags |= MOUSE_MAY_STOP_VIS;
	    }
	    else if (mouse_has(MOUSE_VISUAL))
		jump_flags |= MOUSE_MAY_VIS;
	}
	else if (which_button == MOUSE_RIGHT)
	{
	    if (is_click && VIsual_active)
	    {
		// Remember the start and end of visual before moving the
		// cursor.
		if (LT_POS(curwin->w_cursor, VIsual))
		{
		    start_visual = curwin->w_cursor;
		    end_visual = VIsual;
		}
		else
		{
		    start_visual = VIsual;
		    end_visual = curwin->w_cursor;
		}
	    }
	    jump_flags |= MOUSE_FOCUS;
	    if (mouse_has(MOUSE_VISUAL))
		jump_flags |= MOUSE_MAY_VIS;
	}
    }

    // If an operator is pending, ignore all drags and releases until the
    // next mouse click.
    if (!is_drag && oap != NULL && oap->op_type != OP_NOP)
    {
	got_click = FALSE;
	oap->motion_type = MCHAR;
    }

    // When releasing the button let jump_to_mouse() know.
    if (!is_click && !is_drag)
	jump_flags |= MOUSE_RELEASED;

    // JUMP!
    jump_flags = jump_to_mouse(jump_flags,
			oap == NULL ? NULL : &(oap->inclusive), which_button);

#ifdef FEAT_MENU
    // A click in the window toolbar has no side effects.
    if (jump_flags & MOUSE_WINBAR)
	return FALSE;
#endif
    moved = (jump_flags & CURSOR_MOVED);
    in_status_line = (jump_flags & IN_STATUS_LINE);
    in_sep_line = (jump_flags & IN_SEP_LINE);

#ifdef FEAT_NETBEANS_INTG
    if (isNetbeansBuffer(curbuf)
			    && !(jump_flags & (IN_STATUS_LINE | IN_SEP_LINE)))
    {
	int key = KEY2TERMCAP1(c);

	if (key == (int)KE_LEFTRELEASE || key == (int)KE_MIDDLERELEASE
					       || key == (int)KE_RIGHTRELEASE)
	    netbeans_button_release(which_button);
    }
#endif

    // When jumping to another window, clear a pending operator.  That's a bit
    // friendlier than beeping and not jumping to that window.
    if (curwin != old_curwin && oap != NULL && oap->op_type != OP_NOP)
	clearop(oap);

#ifdef FEAT_FOLDING
    if (mod_mask == 0
	    && !is_drag
	    && (jump_flags & (MOUSE_FOLD_CLOSE | MOUSE_FOLD_OPEN))
	    && which_button == MOUSE_LEFT)
    {
	// open or close a fold at this line
	if (jump_flags & MOUSE_FOLD_OPEN)
	    openFold(curwin->w_cursor.lnum, 1L);
	else
	    closeFold(curwin->w_cursor.lnum, 1L);
	// don't move the cursor if still in the same window
	if (curwin == old_curwin)
	    curwin->w_cursor = save_cursor;
    }
#endif

#if defined(FEAT_CLIPBOARD) && defined(FEAT_CMDWIN)
    if ((jump_flags & IN_OTHER_WIN) && !VIsual_active && clip_star.available)
    {
	clip_modeless(which_button, is_click, is_drag);
	return FALSE;
    }
#endif

    // Set global flag that we are extending the Visual area with mouse
    // dragging; temporarily minimize 'scrolloff'.
    if (VIsual_active && is_drag && get_scrolloff_value())
    {
	// In the very first line, allow scrolling one line
	if (mouse_row == 0)
	    mouse_dragging = 2;
	else
	    mouse_dragging = 1;
    }

    // When dragging the mouse above the window, scroll down.
    if (is_drag && mouse_row < 0 && !in_status_line)
    {
	scroll_redraw(FALSE, 1L);
	mouse_row = 0;
    }

    if (start_visual.lnum)		// right click in visual mode
    {
       // When ALT is pressed make Visual mode blockwise.
       if (mod_mask & MOD_MASK_ALT)
	   VIsual_mode = Ctrl_V;

	// In Visual-block mode, divide the area in four, pick up the corner
	// that is in the quarter that the cursor is in.
	if (VIsual_mode == Ctrl_V)
	{
	    getvcols(curwin, &start_visual, &end_visual, &leftcol, &rightcol);
	    if (curwin->w_curswant > (leftcol + rightcol) / 2)
		end_visual.col = leftcol;
	    else
		end_visual.col = rightcol;
	    if (curwin->w_cursor.lnum >=
				    (start_visual.lnum + end_visual.lnum) / 2)
		end_visual.lnum = start_visual.lnum;

	    // move VIsual to the right column
	    start_visual = curwin->w_cursor;	    // save the cursor pos
	    curwin->w_cursor = end_visual;
	    coladvance(end_visual.col);
	    VIsual = curwin->w_cursor;
	    curwin->w_cursor = start_visual;	    // restore the cursor
	}
	else
	{
	    // If the click is before the start of visual, change the start.
	    // If the click is after the end of visual, change the end.  If
	    // the click is inside the visual, change the closest side.
	    if (LT_POS(curwin->w_cursor, start_visual))
		VIsual = end_visual;
	    else if (LT_POS(end_visual, curwin->w_cursor))
		VIsual = start_visual;
	    else
	    {
		// In the same line, compare column number
		if (end_visual.lnum == start_visual.lnum)
		{
		    if (curwin->w_cursor.col - start_visual.col >
				    end_visual.col - curwin->w_cursor.col)
			VIsual = start_visual;
		    else
			VIsual = end_visual;
		}

		// In different lines, compare line number
		else
		{
		    diff = (curwin->w_cursor.lnum - start_visual.lnum) -
				(end_visual.lnum - curwin->w_cursor.lnum);

		    if (diff > 0)		// closest to end
			VIsual = start_visual;
		    else if (diff < 0)	// closest to start
			VIsual = end_visual;
		    else			// in the middle line
		    {
			if (curwin->w_cursor.col <
					(start_visual.col + end_visual.col) / 2)
			    VIsual = end_visual;
			else
			    VIsual = start_visual;
		    }
		}
	    }
	}
    }
    // If Visual mode started in insert mode, execute "CTRL-O"
    else if ((State & MODE_INSERT) && VIsual_active)
	stuffcharReadbuff(Ctrl_O);

    // Middle mouse click: Put text before cursor.
    if (which_button == MOUSE_MIDDLE)
    {
#ifdef FEAT_CLIPBOARD
	if (clip_star.available && regname == 0)
	    regname = '*';
#endif
	if (yank_register_mline(regname))
	{
	    if (mouse_past_bottom)
		dir = FORWARD;
	}
	else if (mouse_past_eol)
	    dir = FORWARD;

	if (fixindent)
	{
	    c1 = (dir == BACKWARD) ? '[' : ']';
	    c2 = 'p';
	}
	else
	{
	    c1 = (dir == FORWARD) ? 'p' : 'P';
	    c2 = NUL;
	}
	prep_redo(regname, count, NUL, c1, NUL, c2, NUL);

	// Remember where the paste started, so in edit() Insstart can be set
	// to this position
	if (restart_edit != 0)
	    where_paste_started = curwin->w_cursor;
	do_put(regname, NULL, dir, count, fixindent | PUT_CURSEND);
    }

#if defined(FEAT_QUICKFIX)
    // Ctrl-Mouse click or double click in a quickfix window jumps to the
    // error under the mouse pointer.
    else if (((mod_mask & MOD_MASK_CTRL)
		|| (mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_2CLICK)
	    && bt_quickfix(curbuf))
    {
	if (curwin->w_llist_ref == NULL)	// quickfix window
	    do_cmdline_cmd((char_u *)".cc");
	else					// location list window
	    do_cmdline_cmd((char_u *)".ll");
	got_click = FALSE;		// ignore drag&release now
    }
#endif

    // Ctrl-Mouse click (or double click in a help window) jumps to the tag
    // under the mouse pointer.
    else if ((mod_mask & MOD_MASK_CTRL) || (curbuf->b_help
		     && (mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_2CLICK))
    {
	if (State & MODE_INSERT)
	    stuffcharReadbuff(Ctrl_O);
	stuffcharReadbuff(Ctrl_RSB);
	got_click = FALSE;		// ignore drag&release now
    }

    // Shift-Mouse click searches for the next occurrence of the word under
    // the mouse pointer
    else if ((mod_mask & MOD_MASK_SHIFT))
    {
	if ((State & MODE_INSERT) || (VIsual_active && VIsual_select))
	    stuffcharReadbuff(Ctrl_O);
	if (which_button == MOUSE_LEFT)
	    stuffcharReadbuff('*');
	else	// MOUSE_RIGHT
	    stuffcharReadbuff('#');
    }

    // Handle double clicks, unless on status line
    else if (in_status_line)
    {
#ifdef FEAT_MOUSESHAPE
	if ((is_drag || is_click) && !drag_status_line)
	{
	    drag_status_line = TRUE;
	    update_mouseshape(-1);
	}
#endif
    }
    else if (in_sep_line)
    {
#ifdef FEAT_MOUSESHAPE
	if ((is_drag || is_click) && !drag_sep_line)
	{
	    drag_sep_line = TRUE;
	    update_mouseshape(-1);
	}
#endif
    }
    else if ((mod_mask & MOD_MASK_MULTI_CLICK)
				       && (State & (MODE_NORMAL | MODE_INSERT))
	     && mouse_has(MOUSE_VISUAL))
    {
	if (is_click || !VIsual_active)
	{
	    if (VIsual_active)
		orig_cursor = VIsual;
	    else
	    {
		check_visual_highlight();
		VIsual = curwin->w_cursor;
		orig_cursor = VIsual;
		VIsual_active = TRUE;
		VIsual_reselect = TRUE;
		// start Select mode if 'selectmode' contains "mouse"
		may_start_select('o');
		setmouse();
	    }
	    if ((mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_2CLICK)
	    {
		// Double click with ALT pressed makes it blockwise.
		if (mod_mask & MOD_MASK_ALT)
		    VIsual_mode = Ctrl_V;
		else
		    VIsual_mode = 'v';
	    }
	    else if ((mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_3CLICK)
		VIsual_mode = 'V';
	    else if ((mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_4CLICK)
		VIsual_mode = Ctrl_V;
#ifdef FEAT_CLIPBOARD
	    // Make sure the clipboard gets updated.  Needed because start and
	    // end may still be the same, and the selection needs to be owned
	    clip_star.vmode = NUL;
#endif
	}
	// A double click selects a word or a block.
	if ((mod_mask & MOD_MASK_MULTI_CLICK) == MOD_MASK_2CLICK)
	{
	    pos_T	*pos = NULL;
	    int		gc;

	    if (is_click)
	    {
		// If the character under the cursor (skipping white space) is
		// not a word character, try finding a match and select a (),
		// {}, [], #if/#endif, etc. block.
		end_visual = curwin->w_cursor;
		while (gc = gchar_pos(&end_visual), VIM_ISWHITE(gc))
		    inc(&end_visual);
		if (oap != NULL)
		    oap->motion_type = MCHAR;
		if (oap != NULL
			&& VIsual_mode == 'v'
			&& !vim_iswordc(gchar_pos(&end_visual))
			&& EQUAL_POS(curwin->w_cursor, VIsual)
			&& (pos = findmatch(oap, NUL)) != NULL)
		{
		    curwin->w_cursor = *pos;
		    if (oap->motion_type == MLINE)
			VIsual_mode = 'V';
		    else if (*p_sel == 'e')
		    {
			if (LT_POS(curwin->w_cursor, VIsual))
			    ++VIsual.col;
			else
			    ++curwin->w_cursor.col;
		    }
		}
	    }

	    if (pos == NULL && (is_click || is_drag))
	    {
		// When not found a match or when dragging: extend to include
		// a word.
		if (LT_POS(curwin->w_cursor, orig_cursor))
		{
		    find_start_of_word(&curwin->w_cursor);
		    find_end_of_word(&VIsual);
		}
		else
		{
		    find_start_of_word(&VIsual);
		    if (*p_sel == 'e' && *ml_get_cursor() != NUL)
			curwin->w_cursor.col +=
					 (*mb_ptr2len)(ml_get_cursor());
		    find_end_of_word(&curwin->w_cursor);
		}
	    }
	    curwin->w_set_curswant = TRUE;
	}
	if (is_click)
	    redraw_curbuf_later(UPD_INVERTED);	// update the inversion
    }
    else if (VIsual_active && !old_active)
    {
	if (mod_mask & MOD_MASK_ALT)
	    VIsual_mode = Ctrl_V;
	else
	    VIsual_mode = 'v';
    }

    // If Visual mode changed show it later.
    if ((!VIsual_active && old_active && mode_displayed)
	    || (VIsual_active && p_smd && msg_silent == 0
				 && (!old_active || VIsual_mode != old_mode)))
	redraw_cmdline = TRUE;

    return moved;
}


mouse_model_popup(void)
{
    return (p_mousem[0] == 'p');
}


ex_redraw(exarg_T *eap)
{
    redraw_cmd(eap->forceit);
}


do_exmode(
    int		improved)	    // TRUE for "improved Ex" mode
{
    int		save_msg_scroll;
    int		prev_msg_row;
    linenr_T	prev_line;
    varnumber_T	changedtick;

    if (improved)
	exmode_active = EXMODE_VIM;
    else
	exmode_active = EXMODE_NORMAL;
    State = MODE_NORMAL;
    may_trigger_modechanged();

    // When using ":global /pat/ visual" and then "Q" we return to continue
    // the :global command.
    if (global_busy)
	return;

    save_msg_scroll = msg_scroll;
    ++RedrawingDisabled;	    // don't redisplay the window
    ++no_wait_return;		    // don't wait for return
#ifdef FEAT_GUI
    // Ignore scrollbar and mouse events in Ex mode
    ++hold_gui_events;
#endif

    msg(_("Entering Ex mode.  Type \"visual\" to go to Normal mode."));
    while (exmode_active)
    {
	// Check for a ":normal" command and no more characters left.
	if (ex_normal_busy > 0 && typebuf.tb_len == 0)
	{
	    exmode_active = FALSE;
	    break;
	}
	msg_scroll = TRUE;
	need_wait_return = FALSE;
	ex_pressedreturn = FALSE;
	ex_no_reprint = FALSE;
	changedtick = CHANGEDTICK(curbuf);
	prev_msg_row = msg_row;
	prev_line = curwin->w_cursor.lnum;
	if (improved)
	{
	    cmdline_row = msg_row;
	    do_cmdline(NULL, getexline, NULL, 0);
	}
	else
	    do_cmdline(NULL, getexmodeline, NULL, DOCMD_NOWAIT);
	lines_left = Rows - 1;

	if ((prev_line != curwin->w_cursor.lnum
		   || changedtick != CHANGEDTICK(curbuf)) && !ex_no_reprint)
	{
	    if (curbuf->b_ml.ml_flags & ML_EMPTY)
		emsg(_(e_empty_buffer));
	    else
	    {
		if (ex_pressedreturn)
		{
		    // go up one line, to overwrite the ":<CR>" line, so the
		    // output doesn't contain empty lines.
		    msg_row = prev_msg_row;
		    if (prev_msg_row == Rows - 1)
			msg_row--;
		}
		msg_col = 0;
		print_line_no_prefix(curwin->w_cursor.lnum, FALSE, FALSE);
		msg_clr_eos();
	    }
	}
	else if (ex_pressedreturn && !ex_no_reprint)	// must be at EOF
	{
	    if (curbuf->b_ml.ml_flags & ML_EMPTY)
		emsg(_(e_empty_buffer));
	    else
		emsg(_(e_at_end_of_file));
	}
    }

#ifdef FEAT_GUI
    --hold_gui_events;
#endif
    --RedrawingDisabled;
    --no_wait_return;
    update_screen(UPD_CLEAR);
    need_wait_return = FALSE;
    msg_scroll = save_msg_scroll;
}


static char* ReadPassword(rfbClient* client) {
	int i;
	char* p=calloc(1,9);
	if (!p) return p;
#ifndef WIN32
	struct termios save,noecho;
	if(tcgetattr(fileno(stdin),&save)!=0) return p;
	noecho=save; noecho.c_lflag &= ~ECHO;
	if(tcsetattr(fileno(stdin),TCSAFLUSH,&noecho)!=0) return p;
#endif
	fprintf(stderr,"Password: ");
	fflush(stderr);
	i=0;
	while(1) {
		int c=fgetc(stdin);
		if(c=='\n')
			break;
		if(i<8) {
			p[i]=c;
			i++;
			p[i]=0;
		}
	}
#ifndef WIN32
	tcsetattr(fileno(stdin),TCSAFLUSH,&save);
#endif
	return p;
}


static inline unsigned char *PopQuantumLongPixel(QuantumInfo *quantum_info,
  const size_t pixel,unsigned char *magick_restrict pixels)
{
  ssize_t
    i;

  size_t
    quantum_bits;

  if (quantum_info->state.bits == 0U)
    quantum_info->state.bits=32UL;
  for (i=(ssize_t) quantum_info->depth; i > 0; )
  {
    quantum_bits=(size_t) i;
    if (quantum_bits > quantum_info->state.bits)
      quantum_bits=quantum_info->state.bits;
    quantum_info->state.pixel|=(((pixel >> (quantum_info->depth-i)) &
      quantum_info->state.mask[quantum_bits]) << (32U-
        quantum_info->state.bits));
    i-=(ssize_t) quantum_bits;
    quantum_info->state.bits-=quantum_bits;
    if (quantum_info->state.bits == 0U)
      {
        pixels=PopLongPixel(quantum_info->endian,quantum_info->state.pixel,
          pixels);
        quantum_info->state.pixel=0U;
        quantum_info->state.bits=32U;
      }
  }
  return(pixels);
}


static void ExportCMYKQuantum(const Image *image,QuantumInfo *quantum_info,
  const MagickSizeType number_pixels,const Quantum *magick_restrict p,
  unsigned char *magick_restrict q,ExceptionInfo *exception)
{
  ssize_t
    x;

  if (image->colorspace != CMYKColorspace)
    {
      (void) ThrowMagickException(exception,GetMagickModule(),ImageError,
        "ColorSeparatedImageRequired","`%s'",image->filename);
      return;
    }
  switch (quantum_info->depth)
  {
    case 8:
    {
      unsigned char
        pixel;

      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToChar(GetPixelRed(image,p));
        q=PopCharPixel(pixel,q);
        pixel=ScaleQuantumToChar(GetPixelGreen(image,p));
        q=PopCharPixel(pixel,q);
        pixel=ScaleQuantumToChar(GetPixelBlue(image,p));
        q=PopCharPixel(pixel,q);
        pixel=ScaleQuantumToChar(GetPixelBlack(image,p));
        q=PopCharPixel(pixel,q);
        p+=GetPixelChannels(image);
        q+=quantum_info->pad;
      }
      break;
    }
    case 16:
    {
      unsigned short
        pixel;

      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelRed(image,p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelGreen(image,p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelBlue(image,p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelBlack(image,p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            p+=GetPixelChannels(image);
            q+=quantum_info->pad;
          }
          break;
        }
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToShort(GetPixelRed(image,p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToShort(GetPixelGreen(image,p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToShort(GetPixelBlue(image,p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToShort(GetPixelBlack(image,p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        p+=GetPixelChannels(image);
        q+=quantum_info->pad;
      }
      break;
    }
    case 32:
    {
      unsigned int
        pixel;

      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            q=PopFloatPixel(quantum_info,(float) GetPixelRed(image,p),q);
            q=PopFloatPixel(quantum_info,(float) GetPixelGreen(image,p),q);
            q=PopFloatPixel(quantum_info,(float) GetPixelBlue(image,p),q);
            q=PopFloatPixel(quantum_info,(float) GetPixelBlack(image,p),q);
            p+=GetPixelChannels(image);
            q+=quantum_info->pad;
          }
          break;
        }
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToLong(GetPixelRed(image,p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToLong(GetPixelGreen(image,p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToLong(GetPixelBlue(image,p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToLong(GetPixelBlack(image,p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        p+=GetPixelChannels(image);
        q+=quantum_info->pad;
      }
      break;
    }
    case 64:
    {
      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            q=PopDoublePixel(quantum_info,(double) GetPixelRed(image,p),q);
            q=PopDoublePixel(quantum_info,(double) GetPixelGreen(image,p),q);
            q=PopDoublePixel(quantum_info,(double) GetPixelBlue(image,p),q);
            q=PopDoublePixel(quantum_info,(double) GetPixelBlack(image,p),q);
            p+=GetPixelChannels(image);
            q+=quantum_info->pad;
          }
          break;
        }
    }
    default:
    {
      QuantumAny
        range;

      range=GetQuantumRange(quantum_info->depth);
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        q=PopQuantumPixel(quantum_info,ScaleQuantumToAny(GetPixelRed(image,p),
          range),q);
        q=PopQuantumPixel(quantum_info,ScaleQuantumToAny(GetPixelGreen(image,p),
          range),q);
        q=PopQuantumPixel(quantum_info,ScaleQuantumToAny(GetPixelBlue(image,p),
          range),q);
        q=PopQuantumPixel(quantum_info,ScaleQuantumToAny(GetPixelBlack(image,p),
          range),q);
        p+=GetPixelChannels(image);
        q+=quantum_info->pad;
      }
      break;
    }
  }
}


static void ExportOpacityQuantum(const Image *image,QuantumInfo *quantum_info,
  const MagickSizeType number_pixels,const Quantum *magick_restrict p,
  unsigned char *magick_restrict q,ExceptionInfo *exception)
{
  QuantumAny
    range;

  ssize_t
    x;

  assert(exception != (ExceptionInfo *) NULL);
  assert(exception->signature == MagickCoreSignature);
  switch (quantum_info->depth)
  {
    case 8:
    {
      unsigned char
        pixel;

      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToChar(GetPixelOpacity(image,p));
        q=PopCharPixel(pixel,q);
        p+=GetPixelChannels(image);
        q+=quantum_info->pad;
      }
      break;
    }
    case 16:
    {
      unsigned short
        pixel;

      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelOpacity(image,p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            p+=GetPixelChannels(image);
            q+=quantum_info->pad;
          }
          break;
        }
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToShort(GetPixelOpacity(image,p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        p+=GetPixelChannels(image);
        q+=quantum_info->pad;
      }
      break;
    }
    case 32:
    {
      unsigned int
        pixel;

      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            q=PopFloatPixel(quantum_info,(float) GetPixelOpacity(image,p),q);
            p+=GetPixelChannels(image);
            q+=quantum_info->pad;
          }
          break;
        }
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToLong(GetPixelOpacity(image,p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        p+=GetPixelChannels(image);
        q+=quantum_info->pad;
      }
      break;
    }
    case 64:
    {
      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            q=PopDoublePixel(quantum_info,(double) GetPixelOpacity(image,p),q);
            p+=GetPixelChannels(image);
            q+=quantum_info->pad;
          }
          break;
        }
    }
    default:
    {
      range=GetQuantumRange(quantum_info->depth);
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        q=PopQuantumPixel(quantum_info,ScaleQuantumToAny(
          GetPixelOpacity(image,p),range),q);
        p+=GetPixelChannels(image);
        q+=quantum_info->pad;
      }
      break;
    }
  }
}


static void ExportBGRAQuantum(QuantumInfo *quantum_info,
  const MagickSizeType number_pixels,const PixelPacket *magick_restrict p,
  unsigned char *magick_restrict q)
{
  QuantumAny
    range;

  ssize_t
    x;

  switch (quantum_info->depth)
  {
    case 8:
    {
      unsigned char
        pixel;

      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToChar(GetPixelBlue(p));
        q=PopCharPixel(pixel,q);
        pixel=ScaleQuantumToChar(GetPixelGreen(p));
        q=PopCharPixel(pixel,q);
        pixel=ScaleQuantumToChar(GetPixelRed(p));
        q=PopCharPixel(pixel,q);
        pixel=ScaleQuantumToChar((Quantum) GetPixelAlpha(p));
        q=PopCharPixel(pixel,q);
        p++;
        q+=quantum_info->pad;
      }
      break;
    }
    case 10:
    {
      unsigned int
        pixel;

      range=GetQuantumRange(quantum_info->depth);
      if (quantum_info->pack == MagickFalse)
        {
          ssize_t
            i;

          size_t
            quantum;

          ssize_t
            n;

          n=0;
          quantum=0;
          pixel=0;
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            for (i=0; i < 4; i++)
            {
              switch (i)
              {
                case 0: quantum=GetPixelRed(p); break;
                case 1: quantum=GetPixelGreen(p); break;
                case 2: quantum=GetPixelBlue(p); break;
                case 3: quantum=(Quantum) GetPixelAlpha(p); break;
              }
              switch (n % 3)
              {
                case 0:
                {
                  pixel|=(size_t) (ScaleQuantumToAny((Quantum) quantum,
                    range) << 22);
                  break;
                }
                case 1:
                {
                  pixel|=(size_t) (ScaleQuantumToAny((Quantum) quantum,
                    range) << 12);
                  break;
                }
                case 2:
                {
                  pixel|=(size_t) (ScaleQuantumToAny((Quantum) quantum,
                    range) << 2);
                  q=PopLongPixel(quantum_info->endian,pixel,q);
                  pixel=0;
                  break;
                }
              }
              n++;
            }
            p++;
            q+=quantum_info->pad;
          }
          break;
        }
      if (quantum_info->quantum == 32UL)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            pixel=(unsigned int) ScaleQuantumToAny(GetPixelRed(p),range);
            q=PopQuantumLongPixel(quantum_info,pixel,q);
            pixel=(unsigned int) ScaleQuantumToAny(GetPixelGreen(p),range);
            q=PopQuantumLongPixel(quantum_info,pixel,q);
            pixel=(unsigned int) ScaleQuantumToAny(GetPixelBlue(p),range);
            q=PopQuantumLongPixel(quantum_info,pixel,q);
            pixel=(unsigned int) ScaleQuantumToAny((Quantum) GetPixelAlpha(p),
              range);
            q=PopQuantumLongPixel(quantum_info,pixel,q);
            p++;
            q+=quantum_info->pad;
          }
          break;
        }
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=(unsigned int) ScaleQuantumToAny(GetPixelRed(p),range);
        q=PopQuantumPixel(quantum_info,pixel,q);
        pixel=(unsigned int) ScaleQuantumToAny(GetPixelGreen(p),range);
        q=PopQuantumPixel(quantum_info,pixel,q);
        pixel=(unsigned int) ScaleQuantumToAny(GetPixelBlue(p),range);
        q=PopQuantumPixel(quantum_info,pixel,q);
        pixel=(unsigned int) ScaleQuantumToAny((Quantum) GetPixelAlpha(p),
          range);
        q=PopQuantumPixel(quantum_info,pixel,q);
        p++;
        q+=quantum_info->pad;
      }
      break;
    }
    case 16:
    {
      unsigned short
        pixel;

      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelBlue(p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelGreen(p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelRed(p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            pixel=SinglePrecisionToHalf(QuantumScale*GetPixelAlpha(p));
            q=PopShortPixel(quantum_info->endian,pixel,q);
            p++;
            q+=quantum_info->pad;
          }
          break;
        }
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToShort(GetPixelBlue(p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToShort(GetPixelGreen(p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToShort(GetPixelRed(p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToShort((Quantum) GetPixelAlpha(p));
        q=PopShortPixel(quantum_info->endian,pixel,q);
        p++;
        q+=quantum_info->pad;
      }
      break;
    }
    case 32:
    {
      unsigned int
        pixel;

      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            float
              pixel;

            q=PopFloatPixel(quantum_info,(float) GetPixelRed(p),q);
            q=PopFloatPixel(quantum_info,(float) GetPixelGreen(p),q);
            q=PopFloatPixel(quantum_info,(float) GetPixelBlue(p),q);
            pixel=(float) GetPixelAlpha(p);
            q=PopFloatPixel(quantum_info,pixel,q);
            p++;
            q+=quantum_info->pad;
          }
          break;
        }
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        pixel=ScaleQuantumToLong(GetPixelBlue(p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToLong(GetPixelGreen(p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToLong(GetPixelRed(p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        pixel=ScaleQuantumToLong((Quantum) GetPixelAlpha(p));
        q=PopLongPixel(quantum_info->endian,pixel,q);
        p++;
        q+=quantum_info->pad;
      }
      break;
    }
    case 64:
    {
      if (quantum_info->format == FloatingPointQuantumFormat)
        {
          double
            pixel;

          for (x=0; x < (ssize_t) number_pixels; x++)
          {
            q=PopDoublePixel(quantum_info,(double) GetPixelRed(p),q);
            q=PopDoublePixel(quantum_info,(double) GetPixelGreen(p),q);
            q=PopDoublePixel(quantum_info,(double) GetPixelBlue(p),q);
            pixel=(double) GetPixelAlpha(p);
            q=PopDoublePixel(quantum_info,pixel,q);
            p++;
            q+=quantum_info->pad;
          }
          break;
        }
    }
    default:
    {
      range=GetQuantumRange(quantum_info->depth);
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        q=PopQuantumPixel(quantum_info,
          ScaleQuantumToAny(GetPixelBlue(p),range),q);
        q=PopQuantumPixel(quantum_info,
          ScaleQuantumToAny(GetPixelGreen(p),range),q);
        q=PopQuantumPixel(quantum_info,
          ScaleQuantumToAny(GetPixelRed(p),range),q);
        q=PopQuantumPixel(quantum_info,
          ScaleQuantumToAny((Quantum) GetPixelAlpha(p),range),q);
        p++;
        q+=quantum_info->pad;
      }
      break;
    }
  }
}


MagickExport size_t ExportQuantumPixels(const Image *image,
  const CacheView *image_view,const QuantumInfo *quantum_info,
  const QuantumType quantum_type,unsigned char *magick_restrict pixels,
  ExceptionInfo *exception)
{
  MagickRealType
    alpha;

  MagickSizeType
    number_pixels;

  const IndexPacket
    *magick_restrict indexes;

  const PixelPacket
    *magick_restrict p;

  ssize_t
    x;

  unsigned char
    *magick_restrict q;

  size_t
    extent;

  assert(image != (Image *) NULL);
  assert(image->signature == MagickCoreSignature);
  if (image->debug != MagickFalse)
    (void) LogMagickEvent(TraceEvent,GetMagickModule(),"%s",image->filename);
  assert(quantum_info != (QuantumInfo *) NULL);
  assert(quantum_info->signature == MagickCoreSignature);
  if (pixels == (unsigned char *) NULL)
    pixels=GetQuantumPixels(quantum_info);
  if (image_view == (CacheView *) NULL)
    {
      number_pixels=GetImageExtent(image);
      p=GetVirtualPixelQueue(image);
      indexes=GetVirtualIndexQueue(image);
    }
  else
    {
      number_pixels=GetCacheViewExtent(image_view);
      p=GetCacheViewVirtualPixelQueue(image_view);
      indexes=GetCacheViewVirtualIndexQueue(image_view);
    }
  if (quantum_info->alpha_type == AssociatedQuantumAlpha)
    {
      register PixelPacket
        *magick_restrict q;

      /*
        Associate alpha.
      */
      q=GetAuthenticPixelQueue(image);
      if (image_view != (CacheView *) NULL)
        q=(PixelPacket *) GetCacheViewVirtualPixelQueue(image_view);
      for (x=0; x < (ssize_t) image->columns; x++)
      {
        alpha=QuantumScale*GetPixelAlpha(q);
        SetPixelRed(q,ClampToQuantum(alpha*GetPixelRed(q)));
        SetPixelGreen(q,ClampToQuantum(alpha*GetPixelGreen(q)));
        SetPixelBlue(q,ClampToQuantum(alpha*GetPixelBlue(q)));
        q++;
      }
    }
  if ((quantum_type == CbYCrQuantum) || (quantum_type == CbYCrAQuantum))
    {
      Quantum
        quantum;

      register PixelPacket
        *magick_restrict q;

      q=GetAuthenticPixelQueue(image);
      if (image_view != (CacheView *) NULL)
        q=GetAuthenticPixelQueue(image);
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        quantum=GetPixelRed(q);
        SetPixelRed(q,GetPixelGreen(q));
        SetPixelGreen(q,quantum);
        q++;
      }
    }
  x=0;
  q=pixels;
  ResetQuantumState((QuantumInfo *) quantum_info);
  extent=GetQuantumExtent(image,quantum_info,quantum_type);
  switch (quantum_type)
  {
    case AlphaQuantum:
    {
      ExportAlphaQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case BGRQuantum:
    {
      ExportBGRQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case BGRAQuantum:
    {
      ExportBGRAQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case BGROQuantum:
    {
      ExportBGROQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case BlackQuantum:
    {
      ExportBlackQuantum(image,(QuantumInfo *) quantum_info,number_pixels,p,
        indexes,q,exception);
      break;
    }
    case BlueQuantum:
    case YellowQuantum:
    {
      ExportBlueQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case CbYCrYQuantum:
    {
      ExportCbYCrYQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case CMYKQuantum:
    {
      ExportCMYKQuantum(image,(QuantumInfo *) quantum_info,number_pixels,p,
        indexes,q,exception);
      break;
    }
    case CMYKAQuantum:
    {
      ExportCMYKAQuantum(image,(QuantumInfo *) quantum_info,number_pixels,p,
        indexes,q,exception);
      break;
    }
    case CMYKOQuantum:
    {
      ExportCMYKOQuantum(image,(QuantumInfo *) quantum_info,number_pixels,p,
        indexes,q,exception);
      break;
    }
    case GrayQuantum:
    {
      ExportGrayQuantum(image,(QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case GrayAlphaQuantum:
    {
      ExportGrayAlphaQuantum(image,(QuantumInfo *) quantum_info,number_pixels,
        p,q);
      break;
    }
    case GreenQuantum:
    case MagentaQuantum:
    {
      ExportGreenQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case IndexQuantum:
    {
      ExportIndexQuantum(image,(QuantumInfo *) quantum_info,number_pixels,p,
        indexes,q,exception);
      break;
    }
    case IndexAlphaQuantum:
    {
      ExportIndexAlphaQuantum(image,(QuantumInfo *) quantum_info,number_pixels,
        p,indexes,q,exception);
      break;
    }
    case OpacityQuantum:
    {
      ExportOpacityQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case RedQuantum:
    case CyanQuantum:
    {
      ExportRedQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case RGBQuantum:
    case CbYCrQuantum:
    {
      ExportRGBQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case RGBAQuantum:
    case CbYCrAQuantum:
    {
      ExportRGBAQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    case RGBOQuantum:
    {
      ExportRGBOQuantum((QuantumInfo *) quantum_info,number_pixels,p,q);
      break;
    }
    default:
      break;
  }
  if ((quantum_type == CbYCrQuantum) || (quantum_type == CbYCrAQuantum))
    {
      Quantum
        quantum;

      register PixelPacket
        *magick_restrict q;

      q=GetAuthenticPixelQueue(image);
      if (image_view != (CacheView *) NULL)
        q=(PixelPacket *) GetCacheViewVirtualPixelQueue(image_view);
      for (x=0; x < (ssize_t) number_pixels; x++)
      {
        quantum=GetPixelRed(q);
        SetPixelRed(q,GetPixelGreen(q));
        SetPixelGreen(q,quantum);
        q++;
      }
    }
  return(extent);
}


static inline unsigned char *PopQuantumLongPixel(QuantumInfo *quantum_info,
  const size_t pixel,unsigned char *magick_restrict pixels)
{
  ssize_t
    i;

  size_t
    quantum_bits;

  if (quantum_info->state.bits == 0U)
    quantum_info->state.bits=32UL;
  for (i=(ssize_t) quantum_info->depth; i > 0; )
  {
    quantum_bits=(size_t) i;
    if (quantum_bits > quantum_info->state.bits)
      quantum_bits=quantum_info->state.bits;
    quantum_info->state.pixel|=(((pixel >> (quantum_info->depth-i)) &
      quantum_info->state.mask[quantum_bits]) <<
      (32U-quantum_info->state.bits));
    i-=(ssize_t) quantum_bits;
    quantum_info->state.bits-=quantum_bits;
    if (quantum_info->state.bits == 0U)
      {
        pixels=PopLongPixel(quantum_info->endian,quantum_info->state.pixel,
          pixels);
        quantum_info->state.pixel=0U;
        quantum_info->state.bits=32U;
      }
  }
  return(pixels);
}


trace_free(void *arg, void *p_)
{
	trace_info *info = (trace_info *) arg;
	trace_header *p = (trace_header *)p_;

	if (p == NULL)
		return;
	info->current -= p[-1].size;
	free(&p[-1]);
}


static int gettime(void)
{
	static struct timeval first;
	static int once = 1;
	struct timeval now;
	if (once)
	{
		gettimeofday(&first, NULL);
		once = 0;
	}
	gettimeofday(&now, NULL);
	return (now.tv_sec - first.tv_sec) * 1000 + (now.tv_usec - first.tv_usec) / 1000;
}


gc_readl(struct pxa3xx_gcu_priv *priv, unsigned int off)
{
	return __raw_readl(priv->mmio_base + off);
}


static int pxa3xx_gcu_probe(struct platform_device *pdev)
{
	int i, ret, irq;
	struct resource *r;
	struct pxa3xx_gcu_priv *priv;
	struct device *dev = &pdev->dev;

	priv = devm_kzalloc(dev, sizeof(struct pxa3xx_gcu_priv), GFP_KERNEL);
	if (!priv)
		return -ENOMEM;

	init_waitqueue_head(&priv->wait_idle);
	init_waitqueue_head(&priv->wait_free);
	spin_lock_init(&priv->spinlock);

	/* we allocate the misc device structure as part of our own allocation,
	 * so we can get a pointer to our priv structure later on with
	 * container_of(). This isn't really necessary as we have a fixed minor
	 * number anyway, but this is to avoid statics. */

	priv->misc_dev.minor	= PXA3XX_GCU_MINOR,
	priv->misc_dev.name	= DRV_NAME,
	priv->misc_dev.fops	= &pxa3xx_gcu_miscdev_fops;

	/* handle IO resources */
	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	priv->mmio_base = devm_ioremap_resource(dev, r);
	if (IS_ERR(priv->mmio_base))
		return PTR_ERR(priv->mmio_base);

	/* enable the clock */
	priv->clk = devm_clk_get(dev, NULL);
	if (IS_ERR(priv->clk))
		return dev_err_probe(dev, PTR_ERR(priv->clk), "failed to get clock\n");

	/* request the IRQ */
	irq = platform_get_irq(pdev, 0);
	if (irq < 0)
		return irq;

	ret = devm_request_irq(dev, irq, pxa3xx_gcu_handle_irq,
			       0, DRV_NAME, priv);
	if (ret < 0) {
		dev_err(dev, "request_irq failed\n");
		return ret;
	}

	/* allocate dma memory */
	priv->shared = dma_alloc_coherent(dev, SHARED_SIZE,
					  &priv->shared_phys, GFP_KERNEL);
	if (!priv->shared) {
		dev_err(dev, "failed to allocate DMA memory\n");
		return -ENOMEM;
	}

	/* register misc device */
	ret = misc_register(&priv->misc_dev);
	if (ret < 0) {
		dev_err(dev, "misc_register() for minor %d failed\n",
			PXA3XX_GCU_MINOR);
		goto err_free_dma;
	}

	ret = clk_prepare_enable(priv->clk);
	if (ret < 0) {
		dev_err(dev, "failed to enable clock\n");
		goto err_misc_deregister;
	}

	for (i = 0; i < 8; i++) {
		ret = pxa3xx_gcu_add_buffer(dev, priv);
		if (ret) {
			pxa3xx_gcu_free_buffers(dev, priv);
			dev_err(dev, "failed to allocate DMA memory\n");
			goto err_disable_clk;
		}
	}

	platform_set_drvdata(pdev, priv);
	priv->resource_mem = r;
	priv->dev = dev;
	pxa3xx_gcu_reset(priv);
	pxa3xx_gcu_init_debug_timer(priv);

	dev_info(dev, "registered @0x%p, DMA 0x%p (%d bytes), IRQ %d\n",
			(void *) r->start, (void *) priv->shared_phys,
			SHARED_SIZE, irq);
	return 0;

err_disable_clk:
	clk_disable_unprepare(priv->clk);

err_misc_deregister:
	misc_deregister(&priv->misc_dev);

err_free_dma:
	dma_free_coherent(dev, SHARED_SIZE,
			  priv->shared, priv->shared_phys);

	return ret;
}


bool JBIG2Stream::isBinary(bool last) const
{
    return str->isBinary(true);
}


void JBIG2Bitmap::clearToOne()
{
    memset(data, 0xff, h * line);
}


void JBIG2Stream::readExtensionSeg(unsigned int length)
{
    // skip the segment
    byteCounter += curStr->discardChars(length);
}


bool JBIG2Stream::readUByte(unsigned int *x)
{
    int c0;

    if ((c0 = curStr->getChar()) == EOF) {
        return false;
    }
    ++byteCounter;
    *x = (unsigned int)c0;
    return true;
}


void JBIG2Stream::readCodeTableSeg(unsigned int segNum, unsigned int length)
{
    JBIG2HuffmanTable *huffTab;
    unsigned int flags, oob, prefixBits, rangeBits;
    int lowVal, highVal, val;
    unsigned int huffTabSize, i;

    if (!readUByte(&flags) || !readLong(&lowVal) || !readLong(&highVal)) {
        goto eofError;
    }
    oob = flags & 1;
    prefixBits = ((flags >> 1) & 7) + 1;
    rangeBits = ((flags >> 4) & 7) + 1;

    huffDecoder->reset();
    huffTabSize = 8;
    huffTab = (JBIG2HuffmanTable *)gmallocn_checkoverflow(huffTabSize, sizeof(JBIG2HuffmanTable));
    if (unlikely(!huffTab)) {
        goto oomError;
    }

    i = 0;
    val = lowVal;
    while (val < highVal) {
        if (i == huffTabSize) {
            huffTabSize *= 2;
            huffTab = (JBIG2HuffmanTable *)greallocn_checkoverflow(huffTab, huffTabSize, sizeof(JBIG2HuffmanTable));
            if (unlikely(!huffTab)) {
                goto oomError;
            }
        }
        huffTab[i].val = val;
        huffTab[i].prefixLen = huffDecoder->readBits(prefixBits);
        huffTab[i].rangeLen = huffDecoder->readBits(rangeBits);
        if (unlikely(checkedAdd(val, 1 << huffTab[i].rangeLen, &val))) {
            free(huffTab);
            return;
        }
        ++i;
    }
    if (i + oob + 3 > huffTabSize) {
        huffTabSize = i + oob + 3;
        huffTab = (JBIG2HuffmanTable *)greallocn_checkoverflow(huffTab, huffTabSize, sizeof(JBIG2HuffmanTable));
        if (unlikely(!huffTab)) {
            goto oomError;
        }
    }
    huffTab[i].val = lowVal - 1;
    huffTab[i].prefixLen = huffDecoder->readBits(prefixBits);
    huffTab[i].rangeLen = jbig2HuffmanLOW;
    ++i;
    huffTab[i].val = highVal;
    huffTab[i].prefixLen = huffDecoder->readBits(prefixBits);
    huffTab[i].rangeLen = 32;
    ++i;
    if (oob) {
        huffTab[i].val = 0;
        huffTab[i].prefixLen = huffDecoder->readBits(prefixBits);
        huffTab[i].rangeLen = jbig2HuffmanOOB;
        ++i;
    }
    huffTab[i].val = 0;
    huffTab[i].prefixLen = 0;
    huffTab[i].rangeLen = jbig2HuffmanEOT;
    if (JBIG2HuffmanDecoder::buildTable(huffTab, i)) {
        // create and store the new table segment
        segments.push_back(std::make_unique<JBIG2CodeTable>(segNum, huffTab));
    } else {
        free(huffTab);
    }

    return;

eofError:
    error(errSyntaxError, curStr->getPos(), "Unexpected EOF in JBIG2 stream");
oomError:
    error(errInternal, curStr->getPos(), "Failed allocation when processing JBIG2 stream");
}