static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
{
	struct ioat_chan_common *chan = &ioat->base;
	struct ioatdma_device *device = chan->device;
	struct ioat_ring_ent *desc;
	bool seen_current = false;
	int idx = ioat->tail, i;
	u16 active;

	dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
		__func__, ioat->head, ioat->tail, ioat->issued);

	/*
	 * At restart of the channel, the completion address and the
	 * channel status will be 0 due to starting a new chain. Since
	 * it's new chain and the first descriptor "fails", there is
	 * nothing to clean up. We do not want to reap the entire submitted
	 * chain due to this 0 address value and then BUG.
	 */
	if (!phys_complete)
		return;

	active = ioat2_ring_active(ioat);
	for (i = 0; i < active && !seen_current; i++) {
		struct dma_async_tx_descriptor *tx;

		smp_read_barrier_depends();
		prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));
		desc = ioat2_get_ring_ent(ioat, idx + i);
		dump_desc_dbg(ioat, desc);

		/* set err stat if we are using dwbes */
		if (device->cap & IOAT_CAP_DWBES)
			desc_get_errstat(ioat, desc);

		tx = &desc->txd;
		if (tx->cookie) {
			dma_cookie_complete(tx);
			dma_descriptor_unmap(tx);
			if (tx->callback) {
				tx->callback(tx->callback_param);
				tx->callback = NULL;
			}
		}

		if (tx->phys == phys_complete)
			seen_current = true;

		/* skip extended descriptors */
		if (desc_has_ext(desc)) {
			BUG_ON(i + 1 >= active);
			i++;
		}

		/* cleanup super extended descriptors */
		if (desc->sed) {
			ioat3_free_sed(device, desc->sed);
			desc->sed = NULL;
		}
	}
	smp_mb(); /* finish all descriptor reads before incrementing tail */
	ioat->tail = idx + i;
	BUG_ON(active && !seen_current); /* no active descs have written a completion? */
	chan->last_completion = phys_complete;

	if (active - i == 0) {
		dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
			__func__);
		clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
		mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
	}
	/* 5 microsecond delay per pending descriptor */
	writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK),
	       chan->device->reg_base + IOAT_INTRDELAY_OFFSET);
}
static void __cleanup(struct ioat2_dma_chan *ioat, dma_addr_t phys_complete)
{
	struct ioat_chan_common *chan = &ioat->base;
	struct ioatdma_device *device = chan->device;
	struct ioat_ring_ent *desc;
	bool seen_current = false;
	int idx = ioat->tail, i;
	u16 active;

	dev_dbg(to_dev(chan), "%s: head: %#x tail: %#x issued: %#x\n",
		__func__, ioat->head, ioat->tail, ioat->issued);

	 
	if (!phys_complete)
		return;

	active = ioat2_ring_active(ioat);
	for (i = 0; i < active && !seen_current; i++) {
		struct dma_async_tx_descriptor *tx;

		smp_read_barrier_depends();
		prefetch(ioat2_get_ring_ent(ioat, idx + i + 1));
		desc = ioat2_get_ring_ent(ioat, idx + i);
		dump_desc_dbg(ioat, desc);

		 
		if (device->cap & IOAT_CAP_DWBES)
			desc_get_errstat(ioat, desc);

		tx = &desc->txd;
		if (tx->cookie) {
			dma_cookie_complete(tx);
			dma_descriptor_unmap(tx);
			if (tx->callback) {
				tx->callback(tx->callback_param);
				tx->callback = NULL;
			}
		}

		if (tx->phys == phys_complete)
			seen_current = true;

		 
		if (desc_has_ext(desc)) {
			BUG_ON(i + 1 >= active);
			i++;
		}

		 
		if (desc->sed) {
			ioat3_free_sed(device, desc->sed);
			desc->sed = NULL;
		}
	}
	smp_mb();  
	ioat->tail = idx + i;
	BUG_ON(active && !seen_current);  
	chan->last_completion = phys_complete;

	if (active - i == 0) {
		dev_dbg(to_dev(chan), "%s: cancel completion timeout\n",
			__func__);
		clear_bit(IOAT_COMPLETION_PENDING, &chan->state);
		mod_timer(&chan->timer, jiffies + IDLE_TIMEOUT);
	}
	 
	writew(min((5 * (active - i)), IOAT_INTRDELAY_MASK),
	       chan->device->reg_base + IOAT_INTRDELAY_OFFSET);
}