mlx5_tx_burst_single_send(struct mlx5_txq_data *restrict txq,
			  struct rte_mbuf **restrict pkts,
			  unsigned int pkts_n,
			  struct mlx5_txq_local *restrict loc,
			  unsigned int olx)
{
	/*
	 * Subroutine is the part of mlx5_tx_burst_single()
	 * and sends single-segment packet with SEND opcode.
	 */
	assert(loc->elts_free && loc->wqe_free);
	assert(pkts_n > loc->pkts_sent);
	pkts += loc->pkts_sent + 1;
	pkts_n -= loc->pkts_sent;
	for (;;) {
		struct mlx5_wqe *restrict wqe;
		enum mlx5_txcmp_code ret;

		assert(NB_SEGS(loc->mbuf) == 1);
		if (MLX5_TXOFF_CONFIG(INLINE)) {
			unsigned int inlen, vlan = 0;

			inlen = rte_pktmbuf_data_len(loc->mbuf);
			if (MLX5_TXOFF_CONFIG(VLAN) &&
			    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {
				vlan = sizeof(struct rte_vlan_hdr);
				inlen += vlan;
				static_assert((sizeof(struct rte_vlan_hdr) +
					       sizeof(struct rte_ether_hdr)) ==
					       MLX5_ESEG_MIN_INLINE_SIZE,
					       "invalid min inline data size");
			}
			/*
			 * If inlining is enabled at configuration time
			 * the limit must be not less than minimal size.
			 * Otherwise we would do extra check for data
			 * size to avoid crashes due to length overflow.
			 */
			assert(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE);
			if (inlen <= txq->inlen_send) {
				unsigned int seg_n, wqe_n;

				rte_prefetch0(rte_pktmbuf_mtod
						(loc->mbuf, uint8_t *));
				/* Check against minimal length. */
				if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE)
					return MLX5_TXCMP_CODE_ERROR;
				/*
				 * Completely inlined packet data WQE:
				 * - Control Segment, SEND opcode
				 * - Ethernet Segment, no VLAN insertion
				 * - Data inlined, VLAN optionally inserted
				 * - Alignment to MLX5_WSEG_SIZE
				 * Have to estimate amount of WQEBBs
				 */
				seg_n = (inlen + 3 * MLX5_WSEG_SIZE -
					 MLX5_ESEG_MIN_INLINE_SIZE +
					 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
				/* Check if there are enough WQEBBs. */
				wqe_n = (seg_n + 3) / 4;
				if (wqe_n > loc->wqe_free)
					return MLX5_TXCMP_CODE_EXIT;
				wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
				loc->wqe_last = wqe;
				mlx5_tx_cseg_init(txq, loc, wqe, seg_n,
						  MLX5_OPCODE_SEND, olx);
				mlx5_tx_eseg_data(txq, loc, wqe,
						  vlan, inlen, 0, olx);
				txq->wqe_ci += wqe_n;
				loc->wqe_free -= wqe_n;
				/*
				 * Packet data are completely inlined,
				 * free the packet immediately.
				 */
				rte_pktmbuf_free_seg(loc->mbuf);
			} else if ((!MLX5_TXOFF_CONFIG(EMPW) ||
				     MLX5_TXOFF_CONFIG(MPW)) &&
					txq->inlen_mode) {
				/*
				 * If minimal inlining is requested the eMPW
				 * feature should be disabled due to data is
				 * inlined into Ethernet Segment, which can
				 * not contain inlined data for eMPW due to
				 * segment shared for all packets.
				 */
				struct mlx5_wqe_dseg *restrict dseg;
				unsigned int ds;
				uint8_t *dptr;

				/*
				 * The inline-mode settings require
				 * to inline the specified amount of
				 * data bytes to the Ethernet Segment.
				 * We should check the free space in
				 * WQE ring buffer to inline partially.
				 */
				assert(txq->inlen_send >= txq->inlen_mode);
				assert(inlen > txq->inlen_mode);
				assert(txq->inlen_mode >=
						MLX5_ESEG_MIN_INLINE_SIZE);
				/*
				 * Check whether there are enough free WQEBBs:
				 * - Control Segment
				 * - Ethernet Segment
				 * - First Segment of inlined Ethernet data
				 * - ... data continued ...
				 * - Finishing Data Segment of pointer type
				 */
				ds = (MLX5_WQE_CSEG_SIZE +
				      MLX5_WQE_ESEG_SIZE +
				      MLX5_WQE_DSEG_SIZE +
				      txq->inlen_mode -
				      MLX5_ESEG_MIN_INLINE_SIZE +
				      MLX5_WQE_DSEG_SIZE +
				      MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
				if (loc->wqe_free < ((ds + 3) / 4))
					return MLX5_TXCMP_CODE_EXIT;
				/*
				 * Build the ordinary SEND WQE:
				 * - Control Segment
				 * - Ethernet Segment, inline inlen_mode bytes
				 * - Data Segment of pointer type
				 */
				wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
				loc->wqe_last = wqe;
				mlx5_tx_cseg_init(txq, loc, wqe, ds,
						  MLX5_OPCODE_SEND, olx);
				dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan,
							 txq->inlen_mode,
							 0, olx);
				dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) +
				       txq->inlen_mode - vlan;
				inlen -= txq->inlen_mode;
				mlx5_tx_dseg_ptr(txq, loc, dseg,
						 dptr, inlen, olx);
				/*
				 * WQE is built, update the loop parameters
				 * and got to the next packet.
				 */
				txq->wqe_ci += (ds + 3) / 4;
				loc->wqe_free -= (ds + 3) / 4;
				/* We have to store mbuf in elts.*/
				assert(MLX5_TXOFF_CONFIG(INLINE));
				txq->elts[txq->elts_head++ & txq->elts_m] =
						loc->mbuf;
				--loc->elts_free;
			} else {
				uint8_t *dptr;
				unsigned int dlen;

				/*
				 * Partially inlined packet data WQE, we have
				 * some space in title WQEBB, we can fill it
				 * with some packet data. It takes one WQEBB,
				 * it is available, no extra space check:
				 * - Control Segment, SEND opcode
				 * - Ethernet Segment, no VLAN insertion
				 * - MLX5_ESEG_MIN_INLINE_SIZE bytes of Data
				 * - Data Segment, pointer type
				 *
				 * We also get here if VLAN insertion is not
				 * supported by HW, the inline is enabled.
				 */
				wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
				loc->wqe_last = wqe;
				mlx5_tx_cseg_init(txq, loc, wqe, 4,
						  MLX5_OPCODE_SEND, olx);
				mlx5_tx_eseg_dmin(txq, loc, wqe, vlan, olx);
				dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) +
				       MLX5_ESEG_MIN_INLINE_SIZE - vlan;
				/*
				 * The length check is performed above, by
				 * comparing with txq->inlen_send. We should
				 * not get overflow here.
				 */
				assert(inlen > MLX5_ESEG_MIN_INLINE_SIZE);
				dlen = inlen - MLX5_ESEG_MIN_INLINE_SIZE;
				mlx5_tx_dseg_ptr(txq, loc, &wqe->dseg[1],
						 dptr, dlen, olx);
				++txq->wqe_ci;
				--loc->wqe_free;
				/* We have to store mbuf in elts.*/
				assert(MLX5_TXOFF_CONFIG(INLINE));
				txq->elts[txq->elts_head++ & txq->elts_m] =
						loc->mbuf;
				--loc->elts_free;
			}
#ifdef MLX5_PMD_SOFT_COUNTERS
			/* Update sent data bytes counter. */
			txq->stats.obytes += vlan +
					rte_pktmbuf_data_len(loc->mbuf);
#endif
		} else {
			/*
			 * No inline at all, it means the CPU cycles saving
			 * is prioritized at configuration, we should not
			 * copy any packet data to WQE.
			 *
			 * SEND WQE, one WQEBB:
			 * - Control Segment, SEND opcode
			 * - Ethernet Segment, optional VLAN, no inline
			 * - Data Segment, pointer type
			 */
			wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
			loc->wqe_last = wqe;
			mlx5_tx_cseg_init(txq, loc, wqe, 3,
					  MLX5_OPCODE_SEND, olx);
			mlx5_tx_eseg_none(txq, loc, wqe, olx);
			mlx5_tx_dseg_ptr
				(txq, loc, &wqe->dseg[0],
				 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
				 rte_pktmbuf_data_len(loc->mbuf), olx);
			++txq->wqe_ci;
			--loc->wqe_free;
			/*
			 * We should not store mbuf pointer in elts
			 * if no inlining is configured, this is done
			 * by calling routine in a batch copy.
			 */
			assert(!MLX5_TXOFF_CONFIG(INLINE));
			--loc->elts_free;
#ifdef MLX5_PMD_SOFT_COUNTERS
			/* Update sent data bytes counter. */
			txq->stats.obytes += rte_pktmbuf_data_len(loc->mbuf);
			if (MLX5_TXOFF_CONFIG(VLAN) &&
			    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)
				txq->stats.obytes +=
					sizeof(struct rte_vlan_hdr);
#endif
		}
		++loc->pkts_sent;
		--pkts_n;
		if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))
			return MLX5_TXCMP_CODE_EXIT;
		loc->mbuf = *pkts++;
		if (pkts_n > 1)
			rte_prefetch0(*pkts);
		ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
		if (unlikely(ret != MLX5_TXCMP_CODE_SINGLE))
			return ret;
	}
	assert(false);
}
mlx5_tx_burst_single_send(struct mlx5_txq_data *restrict txq,
			  struct rte_mbuf **restrict pkts,
			  unsigned int pkts_n,
			  struct mlx5_txq_local *restrict loc,
			  unsigned int olx)
{
	 
	assert(loc->elts_free && loc->wqe_free);
	assert(pkts_n > loc->pkts_sent);
	pkts += loc->pkts_sent + 1;
	pkts_n -= loc->pkts_sent;
	for (;;) {
		struct mlx5_wqe *restrict wqe;
		enum mlx5_txcmp_code ret;

		assert(NB_SEGS(loc->mbuf) == 1);
		if (MLX5_TXOFF_CONFIG(INLINE)) {
			unsigned int inlen, vlan = 0;

			inlen = rte_pktmbuf_data_len(loc->mbuf);
			if (MLX5_TXOFF_CONFIG(VLAN) &&
			    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {
				vlan = sizeof(struct rte_vlan_hdr);
				inlen += vlan;
				static_assert((sizeof(struct rte_vlan_hdr) +
					       sizeof(struct rte_ether_hdr)) ==
					       MLX5_ESEG_MIN_INLINE_SIZE,
					       "invalid min inline data size");
			}
			 
			assert(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE);
			if (inlen <= txq->inlen_send) {
				unsigned int seg_n, wqe_n;

				rte_prefetch0(rte_pktmbuf_mtod
						(loc->mbuf, uint8_t *));
				 
				if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE)
					return MLX5_TXCMP_CODE_ERROR;
				 
				seg_n = (inlen + 3 * MLX5_WSEG_SIZE -
					 MLX5_ESEG_MIN_INLINE_SIZE +
					 MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
				 
				wqe_n = (seg_n + 3) / 4;
				if (wqe_n > loc->wqe_free)
					return MLX5_TXCMP_CODE_EXIT;
				wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
				loc->wqe_last = wqe;
				mlx5_tx_cseg_init(txq, loc, wqe, seg_n,
						  MLX5_OPCODE_SEND, olx);
				mlx5_tx_eseg_data(txq, loc, wqe,
						  vlan, inlen, 0, olx);
				txq->wqe_ci += wqe_n;
				loc->wqe_free -= wqe_n;
				 
				rte_pktmbuf_free_seg(loc->mbuf);
			} else if ((!MLX5_TXOFF_CONFIG(EMPW) ||
				     MLX5_TXOFF_CONFIG(MPW)) &&
					txq->inlen_mode) {
				 
				struct mlx5_wqe_dseg *restrict dseg;
				unsigned int ds;
				uint8_t *dptr;

				 
				assert(txq->inlen_send >= txq->inlen_mode);
				assert(inlen > txq->inlen_mode);
				assert(txq->inlen_mode >=
						MLX5_ESEG_MIN_INLINE_SIZE);
				 
				ds = (MLX5_WQE_CSEG_SIZE +
				      MLX5_WQE_ESEG_SIZE +
				      MLX5_WQE_DSEG_SIZE +
				      txq->inlen_mode -
				      MLX5_ESEG_MIN_INLINE_SIZE +
				      MLX5_WQE_DSEG_SIZE +
				      MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
				if (loc->wqe_free < ((ds + 3) / 4))
					return MLX5_TXCMP_CODE_EXIT;
				 
				wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
				loc->wqe_last = wqe;
				mlx5_tx_cseg_init(txq, loc, wqe, ds,
						  MLX5_OPCODE_SEND, olx);
				dseg = mlx5_tx_eseg_data(txq, loc, wqe, vlan,
							 txq->inlen_mode,
							 0, olx);
				dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) +
				       txq->inlen_mode - vlan;
				inlen -= txq->inlen_mode;
				mlx5_tx_dseg_ptr(txq, loc, dseg,
						 dptr, inlen, olx);
				 
				txq->wqe_ci += (ds + 3) / 4;
				loc->wqe_free -= (ds + 3) / 4;
				 
				assert(MLX5_TXOFF_CONFIG(INLINE));
				txq->elts[txq->elts_head++ & txq->elts_m] =
						loc->mbuf;
				--loc->elts_free;
			} else {
				uint8_t *dptr;
				unsigned int dlen;

				 
				wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
				loc->wqe_last = wqe;
				mlx5_tx_cseg_init(txq, loc, wqe, 4,
						  MLX5_OPCODE_SEND, olx);
				mlx5_tx_eseg_dmin(txq, loc, wqe, vlan, olx);
				dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *) +
				       MLX5_ESEG_MIN_INLINE_SIZE - vlan;
				 
				assert(inlen > MLX5_ESEG_MIN_INLINE_SIZE);
				dlen = inlen - MLX5_ESEG_MIN_INLINE_SIZE;
				mlx5_tx_dseg_ptr(txq, loc, &wqe->dseg[1],
						 dptr, dlen, olx);
				++txq->wqe_ci;
				--loc->wqe_free;
				 
				assert(MLX5_TXOFF_CONFIG(INLINE));
				txq->elts[txq->elts_head++ & txq->elts_m] =
						loc->mbuf;
				--loc->elts_free;
			}
#ifdef MLX5_PMD_SOFT_COUNTERS
			 
			txq->stats.obytes += vlan +
					rte_pktmbuf_data_len(loc->mbuf);
#endif
		} else {
			 
			wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
			loc->wqe_last = wqe;
			mlx5_tx_cseg_init(txq, loc, wqe, 3,
					  MLX5_OPCODE_SEND, olx);
			mlx5_tx_eseg_none(txq, loc, wqe, olx);
			mlx5_tx_dseg_ptr
				(txq, loc, &wqe->dseg[0],
				 rte_pktmbuf_mtod(loc->mbuf, uint8_t *),
				 rte_pktmbuf_data_len(loc->mbuf), olx);
			++txq->wqe_ci;
			--loc->wqe_free;
			 
			assert(!MLX5_TXOFF_CONFIG(INLINE));
			--loc->elts_free;
#ifdef MLX5_PMD_SOFT_COUNTERS
			 
			txq->stats.obytes += rte_pktmbuf_data_len(loc->mbuf);
			if (MLX5_TXOFF_CONFIG(VLAN) &&
			    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)
				txq->stats.obytes +=
					sizeof(struct rte_vlan_hdr);
#endif
		}
		++loc->pkts_sent;
		--pkts_n;
		if (unlikely(!pkts_n || !loc->elts_free || !loc->wqe_free))
			return MLX5_TXCMP_CODE_EXIT;
		loc->mbuf = *pkts++;
		if (pkts_n > 1)
			rte_prefetch0(*pkts);
		ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
		if (unlikely(ret != MLX5_TXCMP_CODE_SINGLE))
			return ret;
	}
	assert(false);
}