static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)
{
	int ret = 0;
	u16 count;
	unsigned npkts = 0;
	struct user_sdma_txreq *tx = NULL;
	struct hfi1_user_sdma_pkt_q *pq = NULL;
	struct user_sdma_iovec *iovec = NULL;

	if (!req->pq)
		return -EINVAL;

	pq = req->pq;

	/* If tx completion has reported an error, we are done. */
	if (READ_ONCE(req->has_error))
		return -EFAULT;

	/*
	 * Check if we might have sent the entire request already
	 */
	if (unlikely(req->seqnum == req->info.npkts)) {
		if (!list_empty(&req->txps))
			goto dosend;
		return ret;
	}

	if (!maxpkts || maxpkts > req->info.npkts - req->seqnum)
		maxpkts = req->info.npkts - req->seqnum;

	while (npkts < maxpkts) {
		u32 datalen = 0, queued = 0, data_sent = 0;
		u64 iov_offset = 0;

		/*
		 * Check whether any of the completions have come back
		 * with errors. If so, we are not going to process any
		 * more packets from this request.
		 */
		if (READ_ONCE(req->has_error))
			return -EFAULT;

		tx = kmem_cache_alloc(pq->txreq_cache, GFP_KERNEL);
		if (!tx)
			return -ENOMEM;

		tx->flags = 0;
		tx->req = req;
		INIT_LIST_HEAD(&tx->list);

		/*
		 * For the last packet set the ACK request
		 * and disable header suppression.
		 */
		if (req->seqnum == req->info.npkts - 1)
			tx->flags |= (TXREQ_FLAGS_REQ_ACK |
				      TXREQ_FLAGS_REQ_DISABLE_SH);

		/*
		 * Calculate the payload size - this is min of the fragment
		 * (MTU) size or the remaining bytes in the request but only
		 * if we have payload data.
		 */
		if (req->data_len) {
			iovec = &req->iovs[req->iov_idx];
			if (READ_ONCE(iovec->offset) == iovec->iov.iov_len) {
				if (++req->iov_idx == req->data_iovs) {
					ret = -EFAULT;
					goto free_tx;
				}
				iovec = &req->iovs[req->iov_idx];
				WARN_ON(iovec->offset);
			}

			datalen = compute_data_length(req, tx);

			/*
			 * Disable header suppression for the payload <= 8DWS.
			 * If there is an uncorrectable error in the receive
			 * data FIFO when the received payload size is less than
			 * or equal to 8DWS then the RxDmaDataFifoRdUncErr is
			 * not reported.There is set RHF.EccErr if the header
			 * is not suppressed.
			 */
			if (!datalen) {
				SDMA_DBG(req,
					 "Request has data but pkt len is 0");
				ret = -EFAULT;
				goto free_tx;
			} else if (datalen <= 32) {
				tx->flags |= TXREQ_FLAGS_REQ_DISABLE_SH;
			}
		}

		if (req->ahg_idx >= 0) {
			if (!req->seqnum) {
				ret = user_sdma_txadd_ahg(req, tx, datalen);
				if (ret)
					goto free_tx;
			} else {
				int changes;

				changes = set_txreq_header_ahg(req, tx,
							       datalen);
				if (changes < 0) {
					ret = changes;
					goto free_tx;
				}
			}
		} else {
			ret = sdma_txinit(&tx->txreq, 0, sizeof(req->hdr) +
					  datalen, user_sdma_txreq_cb);
			if (ret)
				goto free_tx;
			/*
			 * Modify the header for this packet. This only needs
			 * to be done if we are not going to use AHG. Otherwise,
			 * the HW will do it based on the changes we gave it
			 * during sdma_txinit_ahg().
			 */
			ret = set_txreq_header(req, tx, datalen);
			if (ret)
				goto free_txreq;
		}

		/*
		 * If the request contains any data vectors, add up to
		 * fragsize bytes to the descriptor.
		 */
		while (queued < datalen &&
		       (req->sent + data_sent) < req->data_len) {
			ret = user_sdma_txadd(req, tx, iovec, datalen,
					      &queued, &data_sent, &iov_offset);
			if (ret)
				goto free_txreq;
		}
		/*
		 * The txreq was submitted successfully so we can update
		 * the counters.
		 */
		req->koffset += datalen;
		if (req_opcode(req->info.ctrl) == EXPECTED)
			req->tidoffset += datalen;
		req->sent += data_sent;
		if (req->data_len)
			iovec->offset += iov_offset;
		list_add_tail(&tx->txreq.list, &req->txps);
		/*
		 * It is important to increment this here as it is used to
		 * generate the BTH.PSN and, therefore, can't be bulk-updated
		 * outside of the loop.
		 */
		tx->seqnum = req->seqnum++;
		npkts++;
	}
dosend:
	ret = sdma_send_txlist(req->sde,
			       iowait_get_ib_work(&pq->busy),
			       &req->txps, &count);
	req->seqsubmitted += count;
	if (req->seqsubmitted == req->info.npkts) {
		/*
		 * The txreq has already been submitted to the HW queue
		 * so we can free the AHG entry now. Corruption will not
		 * happen due to the sequential manner in which
		 * descriptors are processed.
		 */
		if (req->ahg_idx >= 0)
			sdma_ahg_free(req->sde, req->ahg_idx);
	}
	return ret;

free_txreq:
	sdma_txclean(pq->dd, &tx->txreq);
free_tx:
	kmem_cache_free(pq->txreq_cache, tx);
	return ret;
}
static int user_sdma_send_pkts(struct user_sdma_request *req, u16 maxpkts)
{
	int ret = 0;
	u16 count;
	unsigned npkts = 0;
	struct user_sdma_txreq *tx = NULL;
	struct hfi1_user_sdma_pkt_q *pq = NULL;
	struct user_sdma_iovec *iovec = NULL;

	if (!req->pq)
		return -EINVAL;

	pq = req->pq;

	 
	if (READ_ONCE(req->has_error))
		return -EFAULT;

	 
	if (unlikely(req->seqnum == req->info.npkts)) {
		if (!list_empty(&req->txps))
			goto dosend;
		return ret;
	}

	if (!maxpkts || maxpkts > req->info.npkts - req->seqnum)
		maxpkts = req->info.npkts - req->seqnum;

	while (npkts < maxpkts) {
		u32 datalen = 0, queued = 0, data_sent = 0;
		u64 iov_offset = 0;

		 
		if (READ_ONCE(req->has_error))
			return -EFAULT;

		tx = kmem_cache_alloc(pq->txreq_cache, GFP_KERNEL);
		if (!tx)
			return -ENOMEM;

		tx->flags = 0;
		tx->req = req;
		INIT_LIST_HEAD(&tx->list);

		 
		if (req->seqnum == req->info.npkts - 1)
			tx->flags |= (TXREQ_FLAGS_REQ_ACK |
				      TXREQ_FLAGS_REQ_DISABLE_SH);

		 
		if (req->data_len) {
			iovec = &req->iovs[req->iov_idx];
			if (READ_ONCE(iovec->offset) == iovec->iov.iov_len) {
				if (++req->iov_idx == req->data_iovs) {
					ret = -EFAULT;
					goto free_tx;
				}
				iovec = &req->iovs[req->iov_idx];
				WARN_ON(iovec->offset);
			}

			datalen = compute_data_length(req, tx);

			 
			if (!datalen) {
				SDMA_DBG(req,
					 "Request has data but pkt len is 0");
				ret = -EFAULT;
				goto free_tx;
			} else if (datalen <= 32) {
				tx->flags |= TXREQ_FLAGS_REQ_DISABLE_SH;
			}
		}

		if (req->ahg_idx >= 0) {
			if (!req->seqnum) {
				ret = user_sdma_txadd_ahg(req, tx, datalen);
				if (ret)
					goto free_tx;
			} else {
				int changes;

				changes = set_txreq_header_ahg(req, tx,
							       datalen);
				if (changes < 0) {
					ret = changes;
					goto free_tx;
				}
			}
		} else {
			ret = sdma_txinit(&tx->txreq, 0, sizeof(req->hdr) +
					  datalen, user_sdma_txreq_cb);
			if (ret)
				goto free_tx;
			 
			ret = set_txreq_header(req, tx, datalen);
			if (ret)
				goto free_txreq;
		}

		 
		while (queued < datalen &&
		       (req->sent + data_sent) < req->data_len) {
			ret = user_sdma_txadd(req, tx, iovec, datalen,
					      &queued, &data_sent, &iov_offset);
			if (ret)
				goto free_txreq;
		}
		 
		req->koffset += datalen;
		if (req_opcode(req->info.ctrl) == EXPECTED)
			req->tidoffset += datalen;
		req->sent += data_sent;
		if (req->data_len)
			iovec->offset += iov_offset;
		list_add_tail(&tx->txreq.list, &req->txps);
		 
		tx->seqnum = req->seqnum++;
		npkts++;
	}
dosend:
	ret = sdma_send_txlist(req->sde,
			       iowait_get_ib_work(&pq->busy),
			       &req->txps, &count);
	req->seqsubmitted += count;
	if (req->seqsubmitted == req->info.npkts) {
		 
		if (req->ahg_idx >= 0)
			sdma_ahg_free(req->sde, req->ahg_idx);
	}
	return ret;

free_txreq:
	sdma_txclean(pq->dd, &tx->txreq);
free_tx:
	kmem_cache_free(pq->txreq_cache, tx);
	return ret;
}