mlx5_tx_burst_empw_inline(struct mlx5_txq_data *__rte_restrict txq,
			  struct rte_mbuf **__rte_restrict pkts,
			  unsigned int pkts_n,
			  struct mlx5_txq_local *__rte_restrict loc,
			  unsigned int olx)
{
	/*
	 * Subroutine is the part of mlx5_tx_burst_single()
	 * and sends single-segment packet with eMPW opcode
	 * with data inlining.
	 */
	MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
	MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW));
	MLX5_ASSERT(loc->elts_free && loc->wqe_free);
	MLX5_ASSERT(pkts_n > loc->pkts_sent);
	static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
	pkts += loc->pkts_sent + 1;
	pkts_n -= loc->pkts_sent;
	for (;;) {
		struct mlx5_wqe_dseg *__rte_restrict dseg;
		struct mlx5_wqe *__rte_restrict wqem;
		enum mlx5_txcmp_code ret;
		unsigned int room, part, nlim;
		unsigned int slen = 0;

		MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
		if (MLX5_TXOFF_CONFIG(TXPP)) {
			enum mlx5_txcmp_code wret;

			/* Generate WAIT for scheduling if requested. */
			wret = mlx5_tx_schedule_send(txq, loc, olx);
			if (wret == MLX5_TXCMP_CODE_EXIT)
				return MLX5_TXCMP_CODE_EXIT;
			if (wret == MLX5_TXCMP_CODE_ERROR)
				return MLX5_TXCMP_CODE_ERROR;
		}
		/*
		 * Limits the amount of packets in one WQE
		 * to improve CQE latency generation.
		 */
		nlim = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ?
				       MLX5_MPW_INLINE_MAX_PACKETS :
				       MLX5_EMPW_MAX_PACKETS);
		/* Check whether we have minimal amount WQEs */
		if (unlikely(loc->wqe_free <
			    ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4)))
			return MLX5_TXCMP_CODE_EXIT;
		if (likely(pkts_n > 1))
			rte_prefetch0(*pkts);
		wqem = txq->wqes + (txq->wqe_ci & txq->wqe_m);
		/*
		 * Build eMPW title WQEBB:
		 * - Control Segment, eMPW opcode, zero DS
		 * - Ethernet Segment, no inline
		 */
		mlx5_tx_cseg_init(txq, loc, wqem, 0,
				  MLX5_OPCODE_ENHANCED_MPSW, olx);
		mlx5_tx_eseg_none(txq, loc, wqem,
				  olx & ~MLX5_TXOFF_CONFIG_VLAN);
		dseg = &wqem->dseg[0];
		/* Store the packet length for legacy MPW. */
		if (MLX5_TXOFF_CONFIG(MPW))
			wqem->eseg.mss = rte_cpu_to_be_16
					 (rte_pktmbuf_data_len(loc->mbuf));
		room = RTE_MIN(MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE,
			       loc->wqe_free) * MLX5_WQE_SIZE -
					MLX5_WQE_CSEG_SIZE -
					MLX5_WQE_ESEG_SIZE;
		/* Limit the room for legacy MPW sessions for performance. */
		if (MLX5_TXOFF_CONFIG(MPW))
			room = RTE_MIN(room,
				       RTE_MAX(txq->inlen_empw +
					       sizeof(dseg->bcount) +
					       (MLX5_TXOFF_CONFIG(VLAN) ?
					       sizeof(struct rte_vlan_hdr) : 0),
					       MLX5_MPW_INLINE_MAX_PACKETS *
					       MLX5_WQE_DSEG_SIZE));
		/* Build WQE till we have space, packets and resources. */
		part = room;
		for (;;) {
			uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf);
			uint8_t *dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);
			unsigned int tlen;

			MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE);
			MLX5_ASSERT((room % MLX5_WQE_DSEG_SIZE) == 0);
			MLX5_ASSERT((uintptr_t)dseg < (uintptr_t)txq->wqes_end);
			/*
			 * Some Tx offloads may cause an error if
			 * packet is not long enough, check against
			 * assumed minimal length.
			 */
			if (unlikely(dlen <= MLX5_ESEG_MIN_INLINE_SIZE)) {
				part -= room;
				if (unlikely(!part))
					return MLX5_TXCMP_CODE_ERROR;
				/*
				 * We have some successfully built
				 * packet Data Segments to send.
				 */
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				return MLX5_TXCMP_CODE_ERROR;
			}
			/* Inline or not inline - that's the Question. */
			if (dlen > txq->inlen_empw ||
			    loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE)
				goto pointer_empw;
			if (MLX5_TXOFF_CONFIG(MPW)) {
				if (dlen > txq->inlen_send)
					goto pointer_empw;
				tlen = dlen;
				if (part == room) {
					/* Open new inline MPW session. */
					tlen += sizeof(dseg->bcount);
					dseg->bcount = RTE_BE32(0);
					dseg = RTE_PTR_ADD
						(dseg, sizeof(dseg->bcount));
				} else {
					/*
					 * No pointer and inline descriptor
					 * intermix for legacy MPW sessions.
					 */
					if (wqem->dseg[0].bcount)
						break;
				}
			} else {
				tlen = sizeof(dseg->bcount) + dlen;
			}
			/* Inline entire packet, optional VLAN insertion. */
			if (MLX5_TXOFF_CONFIG(VLAN) &&
			    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {
				/*
				 * The packet length must be checked in
				 * mlx5_tx_able_to_empw() and packet
				 * fits into inline length guaranteed.
				 */
				MLX5_ASSERT((dlen +
					     sizeof(struct rte_vlan_hdr)) <=
					    txq->inlen_empw);
				tlen += sizeof(struct rte_vlan_hdr);
				if (room < tlen)
					break;
				dseg = mlx5_tx_dseg_vlan(txq, loc, dseg,
							 dptr, dlen, olx);
#ifdef MLX5_PMD_SOFT_COUNTERS
				/* Update sent data bytes counter. */
				slen +=	sizeof(struct rte_vlan_hdr);
#endif
			} else {
				if (room < tlen)
					break;
				dseg = mlx5_tx_dseg_empw(txq, loc, dseg,
							 dptr, dlen, olx);
			}
			if (!MLX5_TXOFF_CONFIG(MPW))
				tlen = RTE_ALIGN(tlen, MLX5_WSEG_SIZE);
			MLX5_ASSERT(room >= tlen);
			room -= tlen;
			/*
			 * Packet data are completely inlined,
			 * free the packet immediately.
			 */
			rte_pktmbuf_free_seg(loc->mbuf);
			goto next_mbuf;
pointer_empw:
			/*
			 * No pointer and inline descriptor
			 * intermix for legacy MPW sessions.
			 */
			if (MLX5_TXOFF_CONFIG(MPW) &&
			    part != room &&
			    wqem->dseg[0].bcount == RTE_BE32(0))
				break;
			/*
			 * Not inlinable VLAN packets are
			 * proceeded outside of this routine.
			 */
			MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE);
			if (MLX5_TXOFF_CONFIG(VLAN))
				MLX5_ASSERT(!(loc->mbuf->ol_flags &
					    PKT_TX_VLAN_PKT));
			mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);
			/* We have to store mbuf in elts.*/
			txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
			room -= MLX5_WQE_DSEG_SIZE;
			/* Ring buffer wraparound is checked at the loop end.*/
			++dseg;
next_mbuf:
#ifdef MLX5_PMD_SOFT_COUNTERS
			/* Update sent data bytes counter. */
			slen += dlen;
#endif
			loc->pkts_sent++;
			loc->elts_free--;
			pkts_n--;
			if (unlikely(!pkts_n || !loc->elts_free)) {
				/*
				 * We have no resources/packets to
				 * continue build descriptors.
				 */
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				return MLX5_TXCMP_CODE_EXIT;
			}
			loc->mbuf = *pkts++;
			if (likely(pkts_n > 1))
				rte_prefetch0(*pkts);
			ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
			/*
			 * Unroll the completion code to avoid
			 * returning variable value - it results in
			 * unoptimized sequent checking in caller.
			 */
			if (ret == MLX5_TXCMP_CODE_MULTI) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_MULTI;
			}
			MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
			if (ret == MLX5_TXCMP_CODE_TSO) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_TSO;
			}
			if (ret == MLX5_TXCMP_CODE_SINGLE) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_SINGLE;
			}
			if (ret != MLX5_TXCMP_CODE_EMPW) {
				MLX5_ASSERT(false);
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				return MLX5_TXCMP_CODE_ERROR;
			}
			/* Check if we have minimal room left. */
			nlim--;
			if (unlikely(!nlim || room < MLX5_WQE_DSEG_SIZE))
				break;
			/*
			 * Check whether packet parameters coincide
			 * within assumed eMPW batch:
			 * - check sum settings
			 * - metadata value
			 * - software parser settings
			 * - packets length (legacy MPW only)
			 * - scheduling is not required
			 */
			if (!mlx5_tx_match_empw(txq, &wqem->eseg,
						loc, dlen, olx))
				break;
			/* Packet attributes match, continue the same eMPW. */
			if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
				dseg = (struct mlx5_wqe_dseg *)txq->wqes;
		}
		/*
		 * We get here to close an existing eMPW
		 * session and start the new one.
		 */
		MLX5_ASSERT(pkts_n);
		part -= room;
		if (unlikely(!part))
			return MLX5_TXCMP_CODE_EXIT;
		mlx5_tx_idone_empw(txq, loc, part, slen, wqem, olx);
		if (unlikely(!loc->elts_free ||
			     !loc->wqe_free))
			return MLX5_TXCMP_CODE_EXIT;
		/* Continue the loop with new eMPW session. */
	}
	MLX5_ASSERT(false);
}
mlx5_tx_burst_empw_inline(struct mlx5_txq_data *__rte_restrict txq,
			  struct rte_mbuf **__rte_restrict pkts,
			  unsigned int pkts_n,
			  struct mlx5_txq_local *__rte_restrict loc,
			  unsigned int olx)
{
	 
	MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
	MLX5_ASSERT(MLX5_TXOFF_CONFIG(EMPW));
	MLX5_ASSERT(loc->elts_free && loc->wqe_free);
	MLX5_ASSERT(pkts_n > loc->pkts_sent);
	static_assert(MLX5_EMPW_MIN_PACKETS >= 2, "invalid min size");
	pkts += loc->pkts_sent + 1;
	pkts_n -= loc->pkts_sent;
	for (;;) {
		struct mlx5_wqe_dseg *__rte_restrict dseg;
		struct mlx5_wqe *__rte_restrict wqem;
		enum mlx5_txcmp_code ret;
		unsigned int room, part, nlim;
		unsigned int slen = 0;

		MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
		if (MLX5_TXOFF_CONFIG(TXPP)) {
			enum mlx5_txcmp_code wret;

			 
			wret = mlx5_tx_schedule_send(txq, loc, olx);
			if (wret == MLX5_TXCMP_CODE_EXIT)
				return MLX5_TXCMP_CODE_EXIT;
			if (wret == MLX5_TXCMP_CODE_ERROR)
				return MLX5_TXCMP_CODE_ERROR;
		}
		 
		nlim = RTE_MIN(pkts_n, MLX5_TXOFF_CONFIG(MPW) ?
				       MLX5_MPW_INLINE_MAX_PACKETS :
				       MLX5_EMPW_MAX_PACKETS);
		 
		if (unlikely(loc->wqe_free <
			    ((2 + MLX5_EMPW_MIN_PACKETS + 3) / 4)))
			return MLX5_TXCMP_CODE_EXIT;
		if (likely(pkts_n > 1))
			rte_prefetch0(*pkts);
		wqem = txq->wqes + (txq->wqe_ci & txq->wqe_m);
		 
		mlx5_tx_cseg_init(txq, loc, wqem, 0,
				  MLX5_OPCODE_ENHANCED_MPSW, olx);
		mlx5_tx_eseg_none(txq, loc, wqem,
				  olx & ~MLX5_TXOFF_CONFIG_VLAN);
		dseg = &wqem->dseg[0];
		 
		if (MLX5_TXOFF_CONFIG(MPW))
			wqem->eseg.mss = rte_cpu_to_be_16
					 (rte_pktmbuf_data_len(loc->mbuf));
		room = RTE_MIN(MLX5_WQE_SIZE_MAX / MLX5_WQE_SIZE,
			       loc->wqe_free) * MLX5_WQE_SIZE -
					MLX5_WQE_CSEG_SIZE -
					MLX5_WQE_ESEG_SIZE;
		 
		if (MLX5_TXOFF_CONFIG(MPW))
			room = RTE_MIN(room,
				       RTE_MAX(txq->inlen_empw +
					       sizeof(dseg->bcount) +
					       (MLX5_TXOFF_CONFIG(VLAN) ?
					       sizeof(struct rte_vlan_hdr) : 0),
					       MLX5_MPW_INLINE_MAX_PACKETS *
					       MLX5_WQE_DSEG_SIZE));
		 
		part = room;
		for (;;) {
			uint32_t dlen = rte_pktmbuf_data_len(loc->mbuf);
			uint8_t *dptr = rte_pktmbuf_mtod(loc->mbuf, uint8_t *);
			unsigned int tlen;

			MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE);
			MLX5_ASSERT((room % MLX5_WQE_DSEG_SIZE) == 0);
			MLX5_ASSERT((uintptr_t)dseg < (uintptr_t)txq->wqes_end);
			 
			if (unlikely(dlen <= MLX5_ESEG_MIN_INLINE_SIZE)) {
				part -= room;
				if (unlikely(!part))
					return MLX5_TXCMP_CODE_ERROR;
				 
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				return MLX5_TXCMP_CODE_ERROR;
			}
			 
			if (dlen > txq->inlen_empw ||
			    loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE)
				goto pointer_empw;
			if (MLX5_TXOFF_CONFIG(MPW)) {
				if (dlen > txq->inlen_send)
					goto pointer_empw;
				tlen = dlen;
				if (part == room) {
					 
					tlen += sizeof(dseg->bcount);
					dseg->bcount = RTE_BE32(0);
					dseg = RTE_PTR_ADD
						(dseg, sizeof(dseg->bcount));
				} else {
					 
					if (wqem->dseg[0].bcount)
						break;
				}
			} else {
				tlen = sizeof(dseg->bcount) + dlen;
			}
			 
			if (MLX5_TXOFF_CONFIG(VLAN) &&
			    loc->mbuf->ol_flags & PKT_TX_VLAN_PKT) {
				 
				MLX5_ASSERT((dlen +
					     sizeof(struct rte_vlan_hdr)) <=
					    txq->inlen_empw);
				tlen += sizeof(struct rte_vlan_hdr);
				if (room < tlen)
					break;
				dseg = mlx5_tx_dseg_vlan(txq, loc, dseg,
							 dptr, dlen, olx);
#ifdef MLX5_PMD_SOFT_COUNTERS
				 
				slen +=	sizeof(struct rte_vlan_hdr);
#endif
			} else {
				if (room < tlen)
					break;
				dseg = mlx5_tx_dseg_empw(txq, loc, dseg,
							 dptr, dlen, olx);
			}
			if (!MLX5_TXOFF_CONFIG(MPW))
				tlen = RTE_ALIGN(tlen, MLX5_WSEG_SIZE);
			MLX5_ASSERT(room >= tlen);
			room -= tlen;
			 
			rte_pktmbuf_free_seg(loc->mbuf);
			goto next_mbuf;
pointer_empw:
			 
			if (MLX5_TXOFF_CONFIG(MPW) &&
			    part != room &&
			    wqem->dseg[0].bcount == RTE_BE32(0))
				break;
			 
			MLX5_ASSERT(room >= MLX5_WQE_DSEG_SIZE);
			if (MLX5_TXOFF_CONFIG(VLAN))
				MLX5_ASSERT(!(loc->mbuf->ol_flags &
					    PKT_TX_VLAN_PKT));
			mlx5_tx_dseg_ptr(txq, loc, dseg, dptr, dlen, olx);
			 
			txq->elts[txq->elts_head++ & txq->elts_m] = loc->mbuf;
			room -= MLX5_WQE_DSEG_SIZE;
			 
			++dseg;
next_mbuf:
#ifdef MLX5_PMD_SOFT_COUNTERS
			 
			slen += dlen;
#endif
			loc->pkts_sent++;
			loc->elts_free--;
			pkts_n--;
			if (unlikely(!pkts_n || !loc->elts_free)) {
				 
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				return MLX5_TXCMP_CODE_EXIT;
			}
			loc->mbuf = *pkts++;
			if (likely(pkts_n > 1))
				rte_prefetch0(*pkts);
			ret = mlx5_tx_able_to_empw(txq, loc, olx, true);
			 
			if (ret == MLX5_TXCMP_CODE_MULTI) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_MULTI;
			}
			MLX5_ASSERT(NB_SEGS(loc->mbuf) == 1);
			if (ret == MLX5_TXCMP_CODE_TSO) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_TSO;
			}
			if (ret == MLX5_TXCMP_CODE_SINGLE) {
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				if (unlikely(!loc->elts_free ||
					     !loc->wqe_free))
					return MLX5_TXCMP_CODE_EXIT;
				return MLX5_TXCMP_CODE_SINGLE;
			}
			if (ret != MLX5_TXCMP_CODE_EMPW) {
				MLX5_ASSERT(false);
				part -= room;
				mlx5_tx_idone_empw(txq, loc, part,
						   slen, wqem, olx);
				return MLX5_TXCMP_CODE_ERROR;
			}
			 
			nlim--;
			if (unlikely(!nlim || room < MLX5_WQE_DSEG_SIZE))
				break;
			 
			if (!mlx5_tx_match_empw(txq, &wqem->eseg,
						loc, dlen, olx))
				break;
			 
			if ((uintptr_t)dseg >= (uintptr_t)txq->wqes_end)
				dseg = (struct mlx5_wqe_dseg *)txq->wqes;
		}
		 
		MLX5_ASSERT(pkts_n);
		part -= room;
		if (unlikely(!part))
			return MLX5_TXCMP_CODE_EXIT;
		mlx5_tx_idone_empw(txq, loc, part, slen, wqem, olx);
		if (unlikely(!loc->elts_free ||
			     !loc->wqe_free))
			return MLX5_TXCMP_CODE_EXIT;
		 
	}
	MLX5_ASSERT(false);
}