mlx5_tx_handle_completion(struct mlx5_txq_data *restrict txq,
			  unsigned int olx __rte_unused)
{
	unsigned int count = MLX5_TX_COMP_MAX_CQE;
	volatile struct mlx5_cqe *last_cqe = NULL;
	bool ring_doorbell = false;
	int ret;

	static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value");
	static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value");
	do {
		volatile struct mlx5_cqe *cqe;

		cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
		ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
			if (likely(ret != MLX5_CQE_STATUS_ERR)) {
				/* No new CQEs in completion queue. */
				assert(ret == MLX5_CQE_STATUS_HW_OWN);
				break;
			}
			/*
			 * Some error occurred, try to restart.
			 * We have no barrier after WQE related Doorbell
			 * written, make sure all writes are completed
			 * here, before we might perform SQ reset.
			 */
			rte_wmb();
			ret = mlx5_tx_error_cqe_handle
				(txq, (volatile struct mlx5_err_cqe *)cqe);
			if (unlikely(ret < 0)) {
				/*
				 * Some error occurred on queue error
				 * handling, we do not advance the index
				 * here, allowing to retry on next call.
				 */
				return;
			}
			/*
			 * We are going to fetch all entries with
			 * MLX5_CQE_SYNDROME_WR_FLUSH_ERR status.
			 * The send queue is supposed to be empty.
			 */
			ring_doorbell = true;
			++txq->cq_ci;
			txq->cq_pi = txq->cq_ci;
			last_cqe = NULL;
			continue;
		}
		/* Normal transmit completion. */
		assert(txq->cq_ci != txq->cq_pi);
		assert((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) ==
			    cqe->wqe_counter);
		ring_doorbell = true;
		++txq->cq_ci;
		last_cqe = cqe;
		/*
		 * We have to restrict the amount of processed CQEs
		 * in one tx_burst routine call. The CQ may be large
		 * and many CQEs may be updated by the NIC in one
		 * transaction. Buffers freeing is time consuming,
		 * multiple iterations may introduce significant
		 * latency.
		 */
		if (likely(--count == 0))
			break;
	} while (true);
	if (likely(ring_doorbell)) {
		/* Ring doorbell to notify hardware. */
		rte_compiler_barrier();
		*txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
		mlx5_tx_comp_flush(txq, last_cqe, olx);
	}
}
mlx5_tx_handle_completion(struct mlx5_txq_data *restrict txq,
			  unsigned int olx __rte_unused)
{
	unsigned int count = MLX5_TX_COMP_MAX_CQE;
	volatile struct mlx5_cqe *last_cqe = NULL;
	bool ring_doorbell = false;
	int ret;

	static_assert(MLX5_CQE_STATUS_HW_OWN < 0, "Must be negative value");
	static_assert(MLX5_CQE_STATUS_SW_OWN < 0, "Must be negative value");
	do {
		volatile struct mlx5_cqe *cqe;

		cqe = &txq->cqes[txq->cq_ci & txq->cqe_m];
		ret = check_cqe(cqe, txq->cqe_s, txq->cq_ci);
		if (unlikely(ret != MLX5_CQE_STATUS_SW_OWN)) {
			if (likely(ret != MLX5_CQE_STATUS_ERR)) {
				 
				assert(ret == MLX5_CQE_STATUS_HW_OWN);
				break;
			}
			 
			rte_wmb();
			ret = mlx5_tx_error_cqe_handle
				(txq, (volatile struct mlx5_err_cqe *)cqe);
			if (unlikely(ret < 0)) {
				 
				return;
			}
			 
			ring_doorbell = true;
			++txq->cq_ci;
			txq->cq_pi = txq->cq_ci;
			last_cqe = NULL;
			continue;
		}
		 
		assert(txq->cq_ci != txq->cq_pi);
		assert((txq->fcqs[txq->cq_ci & txq->cqe_m] >> 16) ==
			    cqe->wqe_counter);
		ring_doorbell = true;
		++txq->cq_ci;
		last_cqe = cqe;
		 
		if (likely(--count == 0))
			break;
	} while (true);
	if (likely(ring_doorbell)) {
		 
		rte_compiler_barrier();
		*txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
		mlx5_tx_comp_flush(txq, last_cqe, olx);
	}
}