mlx5_tx_packet_multi_inline(struct mlx5_txq_data *__rte_restrict txq,
			    struct mlx5_txq_local *__rte_restrict loc,
			    unsigned int olx)
{
	struct mlx5_wqe *__rte_restrict wqe;
	unsigned int ds, inlen, dlen, vlan = 0;

	MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
	MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1);
	if (MLX5_TXOFF_CONFIG(TXPP)) {
		enum mlx5_txcmp_code wret;

		/* Generate WAIT for scheduling if requested. */
		wret = mlx5_tx_schedule_send(txq, loc, olx);
		if (wret == MLX5_TXCMP_CODE_EXIT)
			return MLX5_TXCMP_CODE_EXIT;
		if (wret == MLX5_TXCMP_CODE_ERROR)
			return MLX5_TXCMP_CODE_ERROR;
	}
	/*
	 * First calculate data length to be inlined
	 * to estimate the required space for WQE.
	 */
	dlen = rte_pktmbuf_pkt_len(loc->mbuf);
	if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)
		vlan = sizeof(struct rte_vlan_hdr);
	inlen = dlen + vlan;
	/* Check against minimal length. */
	if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE)
		return MLX5_TXCMP_CODE_ERROR;
	MLX5_ASSERT(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE);
	if (inlen > txq->inlen_send ||
	    loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) {
		struct rte_mbuf *mbuf;
		unsigned int nxlen;
		uintptr_t start;

		mbuf = loc->mbuf;
		nxlen = rte_pktmbuf_data_len(mbuf);
		/*
		 * Packet length exceeds the allowed inline
		 * data length, check whether the minimal
		 * inlining is required.
		 */
		if (txq->inlen_mode) {
			MLX5_ASSERT(txq->inlen_mode >=
				    MLX5_ESEG_MIN_INLINE_SIZE);
			MLX5_ASSERT(txq->inlen_mode <= txq->inlen_send);
			inlen = RTE_MIN(txq->inlen_mode, inlen);
		} else if (vlan && !txq->vlan_en) {
			/*
			 * VLAN insertion is requested and hardware does not
			 * support the offload, will do with software inline.
			 */
			inlen = MLX5_ESEG_MIN_INLINE_SIZE;
		} else if (mbuf->ol_flags & PKT_TX_DYNF_NOINLINE ||
			   nxlen > txq->inlen_send) {
			return mlx5_tx_packet_multi_send(txq, loc, olx);
		} else {
			goto do_first;
		}
		if (mbuf->ol_flags & PKT_TX_DYNF_NOINLINE)
			goto do_build;
		/*
		 * Now we know the minimal amount of data is requested
		 * to inline. Check whether we should inline the buffers
		 * from the chain beginning to eliminate some mbufs.
		 */
		if (unlikely(nxlen <= txq->inlen_send)) {
			/* We can inline first mbuf at least. */
			if (nxlen < inlen) {
				unsigned int smlen;

				/* Scan mbufs till inlen filled. */
				do {
					smlen = nxlen;
					mbuf = NEXT(mbuf);
					MLX5_ASSERT(mbuf);
					nxlen = rte_pktmbuf_data_len(mbuf);
					nxlen += smlen;
				} while (unlikely(nxlen < inlen));
				if (unlikely(nxlen > txq->inlen_send)) {
					/* We cannot inline entire mbuf. */
					smlen = inlen - smlen;
					start = rte_pktmbuf_mtod_offset
						    (mbuf, uintptr_t, smlen);
					goto do_align;
				}
			}
do_first:
			do {
				inlen = nxlen;
				mbuf = NEXT(mbuf);
				/* There should be not end of packet. */
				MLX5_ASSERT(mbuf);
				if (mbuf->ol_flags & PKT_TX_DYNF_NOINLINE)
					break;
				nxlen = inlen + rte_pktmbuf_data_len(mbuf);
			} while (unlikely(nxlen < txq->inlen_send));
		}
		start = rte_pktmbuf_mtod(mbuf, uintptr_t);
		/*
		 * Check whether we can do inline to align start
		 * address of data buffer to cacheline.
		 */
do_align:
		start = (~start + 1) & (RTE_CACHE_LINE_SIZE - 1);
		if (unlikely(start)) {
			start += inlen;
			if (start <= txq->inlen_send)
				inlen = start;
		}
	}
	/*
	 * Check whether there are enough free WQEBBs:
	 * - Control Segment
	 * - Ethernet Segment
	 * - First Segment of inlined Ethernet data
	 * - ... data continued ...
	 * - Data Segments of pointer/min inline type
	 *
	 * Estimate the number of Data Segments conservatively,
	 * supposing no any mbufs is being freed during inlining.
	 */
do_build:
	MLX5_ASSERT(inlen <= txq->inlen_send);
	ds = NB_SEGS(loc->mbuf) + 2 + (inlen -
				       MLX5_ESEG_MIN_INLINE_SIZE +
				       MLX5_WSEG_SIZE +
				       MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
	if (unlikely(loc->wqe_free < ((ds + 3) / 4)))
		return MLX5_TXCMP_CODE_EXIT;
	/* Check for maximal WQE size. */
	if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ds))
		return MLX5_TXCMP_CODE_ERROR;
#ifdef MLX5_PMD_SOFT_COUNTERS
	/* Update sent data bytes/packets counters. */
	txq->stats.obytes += dlen + vlan;
#endif
	wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
	loc->wqe_last = wqe;
	mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_SEND, olx);
	ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 0, olx);
	wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);
	txq->wqe_ci += (ds + 3) / 4;
	loc->wqe_free -= (ds + 3) / 4;
	return MLX5_TXCMP_CODE_MULTI;
}
mlx5_tx_packet_multi_inline(struct mlx5_txq_data *__rte_restrict txq,
			    struct mlx5_txq_local *__rte_restrict loc,
			    unsigned int olx)
{
	struct mlx5_wqe *__rte_restrict wqe;
	unsigned int ds, inlen, dlen, vlan = 0;

	MLX5_ASSERT(MLX5_TXOFF_CONFIG(INLINE));
	MLX5_ASSERT(NB_SEGS(loc->mbuf) > 1);
	if (MLX5_TXOFF_CONFIG(TXPP)) {
		enum mlx5_txcmp_code wret;

		 
		wret = mlx5_tx_schedule_send(txq, loc, olx);
		if (wret == MLX5_TXCMP_CODE_EXIT)
			return MLX5_TXCMP_CODE_EXIT;
		if (wret == MLX5_TXCMP_CODE_ERROR)
			return MLX5_TXCMP_CODE_ERROR;
	}
	 
	dlen = rte_pktmbuf_pkt_len(loc->mbuf);
	if (MLX5_TXOFF_CONFIG(VLAN) && loc->mbuf->ol_flags & PKT_TX_VLAN_PKT)
		vlan = sizeof(struct rte_vlan_hdr);
	inlen = dlen + vlan;
	 
	if (inlen <= MLX5_ESEG_MIN_INLINE_SIZE)
		return MLX5_TXCMP_CODE_ERROR;
	MLX5_ASSERT(txq->inlen_send >= MLX5_ESEG_MIN_INLINE_SIZE);
	if (inlen > txq->inlen_send ||
	    loc->mbuf->ol_flags & PKT_TX_DYNF_NOINLINE) {
		struct rte_mbuf *mbuf;
		unsigned int nxlen;
		uintptr_t start;

		mbuf = loc->mbuf;
		nxlen = rte_pktmbuf_data_len(mbuf);
		 
		if (txq->inlen_mode) {
			MLX5_ASSERT(txq->inlen_mode >=
				    MLX5_ESEG_MIN_INLINE_SIZE);
			MLX5_ASSERT(txq->inlen_mode <= txq->inlen_send);
			inlen = RTE_MIN(txq->inlen_mode, inlen);
		} else if (vlan && !txq->vlan_en) {
			 
			inlen = MLX5_ESEG_MIN_INLINE_SIZE;
		} else if (mbuf->ol_flags & PKT_TX_DYNF_NOINLINE ||
			   nxlen > txq->inlen_send) {
			return mlx5_tx_packet_multi_send(txq, loc, olx);
		} else {
			goto do_first;
		}
		if (mbuf->ol_flags & PKT_TX_DYNF_NOINLINE)
			goto do_build;
		 
		if (unlikely(nxlen <= txq->inlen_send)) {
			 
			if (nxlen < inlen) {
				unsigned int smlen;

				 
				do {
					smlen = nxlen;
					mbuf = NEXT(mbuf);
					MLX5_ASSERT(mbuf);
					nxlen = rte_pktmbuf_data_len(mbuf);
					nxlen += smlen;
				} while (unlikely(nxlen < inlen));
				if (unlikely(nxlen > txq->inlen_send)) {
					 
					smlen = inlen - smlen;
					start = rte_pktmbuf_mtod_offset
						    (mbuf, uintptr_t, smlen);
					goto do_align;
				}
			}
do_first:
			do {
				inlen = nxlen;
				mbuf = NEXT(mbuf);
				 
				MLX5_ASSERT(mbuf);
				if (mbuf->ol_flags & PKT_TX_DYNF_NOINLINE)
					break;
				nxlen = inlen + rte_pktmbuf_data_len(mbuf);
			} while (unlikely(nxlen < txq->inlen_send));
		}
		start = rte_pktmbuf_mtod(mbuf, uintptr_t);
		 
do_align:
		start = (~start + 1) & (RTE_CACHE_LINE_SIZE - 1);
		if (unlikely(start)) {
			start += inlen;
			if (start <= txq->inlen_send)
				inlen = start;
		}
	}
	 
do_build:
	MLX5_ASSERT(inlen <= txq->inlen_send);
	ds = NB_SEGS(loc->mbuf) + 2 + (inlen -
				       MLX5_ESEG_MIN_INLINE_SIZE +
				       MLX5_WSEG_SIZE +
				       MLX5_WSEG_SIZE - 1) / MLX5_WSEG_SIZE;
	if (unlikely(loc->wqe_free < ((ds + 3) / 4)))
		return MLX5_TXCMP_CODE_EXIT;
	 
	if (unlikely((MLX5_WQE_SIZE_MAX / MLX5_WSEG_SIZE) < ds))
		return MLX5_TXCMP_CODE_ERROR;
#ifdef MLX5_PMD_SOFT_COUNTERS
	 
	txq->stats.obytes += dlen + vlan;
#endif
	wqe = txq->wqes + (txq->wqe_ci & txq->wqe_m);
	loc->wqe_last = wqe;
	mlx5_tx_cseg_init(txq, loc, wqe, 0, MLX5_OPCODE_SEND, olx);
	ds = mlx5_tx_mseg_build(txq, loc, wqe, vlan, inlen, 0, olx);
	wqe->cseg.sq_ds = rte_cpu_to_be_32(txq->qp_num_8s | ds);
	txq->wqe_ci += (ds + 3) / 4;
	loc->wqe_free -= (ds + 3) / 4;
	return MLX5_TXCMP_CODE_MULTI;
}