CVE-2020-27673: Insufficient Information (NVD-CWE-noinfo)
value: An issue was discovered in the Linux kernel through 5.9.1, as used with Xen through 4.14.x. Guest OS users can cause a denial of service (host OS hang) via a high rate of events to dom0, aka CID-e99502f76271.
CWE description: There is insufficient information about the issue to classify it; details are unkown or unspecified.
CVSS v3 base score: 5.5
Published: 2020-10-22T21:15Z
Repository: https://github.com/torvalds/linux
Language: C
Files changed: Documentation/admin-guide/kernel-parameters.txt, drivers/xen/events/events_2l.c, drivers/xen/events/events_base.c, drivers/xen/events/events_fifo.c, drivers/xen/events/events_internal.h
Commit date: 2020-10-20 10:22:16 +0200
Commit message: xen/events: defer eoi in case of excessive number of events

In case rogue guests are sending events at high frequency it might
happen that xen_evtchn_do_upcall() won't stop processing events in
dom0. As this is done in irq handling a crash might be the result.

In order to avoid that, delay further inter-domain events after some
time in xen_evtchn_do_upcall() by forcing eoi processing into a
worker on the same cpu, thus inhibiting new events coming in.

The time after which eoi processing is to be delayed is configurable
via a new module parameter "event_loop_timeout" which specifies the
maximum event loop time in jiffies (default: 2, the value was chosen
after some tests showing that a value of 2 was the lowest with an
only slight drop of dom0 network throughput while multiple guests
performed an event storm).

How long eoi processing will be delayed can be specified via another
parameter "event_eoi_delay" (again in jiffies, default 10, again the
value was chosen after testing with different delay values).

This is part of XSA-332.

Cc: stable@vger.kernel.org
Reported-by: Julien Grall <julien@xen.org>
Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>
Reviewed-by: Wei Liu <wl@xen.org>

Diff: diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index a1068742a6df..89d977f0b786 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -5826,10 +5826,18 @@
 			timers (default is 100000). This adjusts the minimum
 			delta of virtualized Xen timers, where lower values
 			improve timer resolution at the expense of processing
 			more timer interrupts.
 
+	xen.event_eoi_delay=	[XEN]
+			How long to delay EOI handling in case of event
+			storms (jiffies). Default is 10.
+
+	xen.event_loop_timeout=	[XEN]
+			After which time (jiffies) the event handling loop
+			should start to delay EOI handling. Default is 2.
+
 	nopv=		[X86,XEN,KVM,HYPER_V,VMWARE]
 			Disables the PV optimizations forcing the guest to run
 			as generic guest with no PV drivers. Currently support
 			XEN HVM, KVM, HYPER_V and VMWARE guest.
 
diff --git a/drivers/xen/events/events_2l.c b/drivers/xen/events/events_2l.c
index e1af5e093ff4..fe5ad0e89cd8 100644
--- a/drivers/xen/events/events_2l.c
+++ b/drivers/xen/events/events_2l.c
@@ -159,11 +159,11 @@ static inline xen_ulong_t active_evtchns(unsigned int cpu,
  *
  * Xen uses a two-level bitmap to speed searching.  The first level is
  * a bitset of words which contain pending event bits.  The second
  * level is a bitset of pending events themselves.
  */
-static void evtchn_2l_handle_events(unsigned cpu)
+static void evtchn_2l_handle_events(unsigned cpu, struct evtchn_loop_ctrl *ctrl)
 {
 	int irq;
 	xen_ulong_t pending_words;
 	xen_ulong_t pending_bits;
 	int start_word_idx, start_bit_idx;
@@ -240,14 +240,11 @@ static void evtchn_2l_handle_events(unsigned cpu)
 
 			bit_idx = EVTCHN_FIRST_BIT(bits);
 
 			/* Process port. */
 			port = (word_idx * BITS_PER_EVTCHN_WORD) + bit_idx;
-			irq = get_evtchn_to_irq(port);
-
-			if (irq != -1)
-				generic_handle_irq(irq);
+			handle_irq_for_port(port, ctrl);
 
 			bit_idx = (bit_idx + 1) % BITS_PER_EVTCHN_WORD;
 
 			/* Next caller starts at last processed + 1 */
 			__this_cpu_write(current_word_idx,
diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c
index 9cbfea5e9a08..cde096a6f11d 100644
--- a/drivers/xen/events/events_base.c
+++ b/drivers/xen/events/events_base.c
@@ -33,10 +33,12 @@
 #include <linux/slab.h>
 #include <linux/irqnr.h>
 #include <linux/pci.h>
 #include <linux/spinlock.h>
 #include <linux/cpuhotplug.h>
+#include <linux/atomic.h>
+#include <linux/ktime.h>
 
 #ifdef CONFIG_X86
 #include <asm/desc.h>
 #include <asm/ptrace.h>
 #include <asm/idtentry.h>
@@ -63,10 +65,19 @@
 #include <xen/interface/vcpu.h>
 #include <asm/hw_irq.h>
 
 #include "events_internal.h"
 
+#undef MODULE_PARAM_PREFIX
+#define MODULE_PARAM_PREFIX "xen."
+
+static uint __read_mostly event_loop_timeout = 2;
+module_param(event_loop_timeout, uint, 0644);
+
+static uint __read_mostly event_eoi_delay = 10;
+module_param(event_eoi_delay, uint, 0644);
+
 const struct evtchn_ops *evtchn_ops;
 
 /*
  * This lock protects updates to the following mapping and reference-count
  * arrays. The lock does not need to be acquired to read the mapping tables.
@@ -86,10 +97,11 @@ static DEFINE_RWLOCK(evtchn_rwlock);
  * Lock hierarchy:
  *
  * irq_mapping_update_lock
  *   evtchn_rwlock
  *     IRQ-desc lock
+ *       percpu eoi_list_lock
  */
 
 static LIST_HEAD(xen_irq_list_head);
 
 /* IRQ <-> VIRQ mapping. */
@@ -118,10 +130,12 @@ static struct irq_chip xen_lateeoi_chip;
 static struct irq_chip xen_percpu_chip;
 static struct irq_chip xen_pirq_chip;
 static void enable_dynirq(struct irq_data *data);
 static void disable_dynirq(struct irq_data *data);
 
+static DEFINE_PER_CPU(unsigned int, irq_epoch);
+
 static void clear_evtchn_to_irq_row(unsigned row)
 {
 	unsigned col;
 
 	for (col = 0; col < EVTCHN_PER_ROW; col++)
@@ -397,21 +411,124 @@ void notify_remote_via_irq(int irq)
 	if (VALID_EVTCHN(evtchn))
 		notify_remote_via_evtchn(evtchn);
 }
 EXPORT_SYMBOL_GPL(notify_remote_via_irq);
 
+struct lateeoi_work {
+	struct delayed_work delayed;
+	spinlock_t eoi_list_lock;
+	struct list_head eoi_list;
+};
+
+static DEFINE_PER_CPU(struct lateeoi_work, lateeoi);
+
+static void lateeoi_list_del(struct irq_info *info)
+{
+	struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
+	unsigned long flags;
+
+	spin_lock_irqsave(&eoi->eoi_list_lock, flags);
+	list_del_init(&info->eoi_list);
+	spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
+}
+
+static void lateeoi_list_add(struct irq_info *info)
+{
+	struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
+	struct irq_info *elem;
+	u64 now = get_jiffies_64();
+	unsigned long delay;
+	unsigned long flags;
+
+	if (now < info->eoi_time)
+		delay = info->eoi_time - now;
+	else
+		delay = 1;
+
+	spin_lock_irqsave(&eoi->eoi_list_lock, flags);
+
+	if (list_empty(&eoi->eoi_list)) {
+		list_add(&info->eoi_list, &eoi->eoi_list);
+		mod_delayed_work_on(info->eoi_cpu, system_wq,
+				    &eoi->delayed, delay);
+	} else {
+		list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) {
+			if (elem->eoi_time <= info->eoi_time)
+				break;
+		}
+		list_add(&info->eoi_list, &elem->eoi_list);
+	}
+
+	spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
+}
+
 static void xen_irq_lateeoi_locked(struct irq_info *info)
 {
 	evtchn_port_t evtchn;
+	unsigned int cpu;
 
 	evtchn = info->evtchn;
-	if (!VALID_EVTCHN(evtchn))
+	if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list))
 		return;
 
+	cpu = info->eoi_cpu;
+	if (info->eoi_time && info->irq_epoch == per_cpu(irq_epoch, cpu)) {
+		lateeoi_list_add(info);
+		return;
+	}
+
+	info->eoi_time = 0;
 	unmask_evtchn(evtchn);
 }
 
+static void xen_irq_lateeoi_worker(struct work_struct *work)
+{
+	struct lateeoi_work *eoi;
+	struct irq_info *info;
+	u64 now = get_jiffies_64();
+	unsigned long flags;
+
+	eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed);
+
+	read_lock_irqsave(&evtchn_rwlock, flags);
+
+	while (true) {
+		spin_lock(&eoi->eoi_list_lock);
+
+		info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info,
+						eoi_list);
+
+		if (info == NULL || now < info->eoi_time) {
+			spin_unlock(&eoi->eoi_list_lock);
+			break;
+		}
+
+		list_del_init(&info->eoi_list);
+
+		spin_unlock(&eoi->eoi_list_lock);
+
+		info->eoi_time = 0;
+
+		xen_irq_lateeoi_locked(info);
+	}
+
+	if (info)
+		mod_delayed_work_on(info->eoi_cpu, system_wq,
+				    &eoi->delayed, info->eoi_time - now);
+
+	read_unlock_irqrestore(&evtchn_rwlock, flags);
+}
+
+static void xen_cpu_init_eoi(unsigned int cpu)
+{
+	struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu);
+
+	INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker);
+	spin_lock_init(&eoi->eoi_list_lock);
+	INIT_LIST_HEAD(&eoi->eoi_list);
+}
+
 void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags)
 {
 	struct irq_info *info;
 	unsigned long flags;
 
@@ -427,10 +544,11 @@ void xen_irq_lateeoi(unsigned int irq, unsigned int eoi_flags)
 EXPORT_SYMBOL_GPL(xen_irq_lateeoi);
 
 static void xen_irq_init(unsigned irq)
 {
 	struct irq_info *info;
+
 #ifdef CONFIG_SMP
 	/* By default all event channels notify CPU#0. */
 	cpumask_copy(irq_get_affinity_mask(irq), cpumask_of(0));
 #endif
 
@@ -441,10 +559,11 @@ static void xen_irq_init(unsigned irq)
 	info->type = IRQT_UNBOUND;
 	info->refcnt = -1;
 
 	set_info_for_irq(irq, info);
 
+	INIT_LIST_HEAD(&info->eoi_list);
 	list_add_tail(&info->list, &xen_irq_list_head);
 }
 
 static int __must_check xen_allocate_irqs_dynamic(int nvec)
 {
@@ -496,10 +615,13 @@ static void xen_free_irq(unsigned irq)
 	if (WARN_ON(!info))
 		return;
 
 	write_lock_irqsave(&evtchn_rwlock, flags);
 
+	if (!list_empty(&info->eoi_list))
+		lateeoi_list_del(info);
+
 	list_del(&info->list);
 
 	set_info_for_irq(irq, NULL);
 
 	WARN_ON(info->refcnt > 0);
@@ -1356,29 +1478,85 @@ void xen_send_IPI_one(unsigned int cpu, enum ipi_vector vector)
 	irq = per_cpu(ipi_to_irq, cpu)[vector];
 	BUG_ON(irq < 0);
 	notify_remote_via_irq(irq);
 }
 
+struct evtchn_loop_ctrl {
+	ktime_t timeout;
+	unsigned count;
+	bool defer_eoi;
+};
+
+void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl)
+{
+	int irq;
+	struct irq_info *info;
+
+	irq = get_evtchn_to_irq(port);
+	if (irq == -1)
+		return;
+
+	/*
+	 * Check for timeout every 256 events.
+	 * We are setting the timeout value only after the first 256
+	 * events in order to not hurt the common case of few loop
+	 * iterations. The 256 is basically an arbitrary value.
+	 *
+	 * In case we are hitting the timeout we need to defer all further
+	 * EOIs in order to ensure to leave the event handling loop rather
+	 * sooner than later.
+	 */
+	if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) {
+		ktime_t kt = ktime_get();
+
+		if (!ctrl->timeout) {
+			kt = ktime_add_ms(kt,
+					  jiffies_to_msecs(event_loop_timeout));
+			ctrl->timeout = kt;
+		} else if (kt > ctrl->timeout) {
+			ctrl->defer_eoi = true;
+		}
+	}
+
+	info = info_for_irq(irq);
+
+	if (ctrl->defer_eoi) {
+		info->eoi_cpu = smp_processor_id();
+		info->irq_epoch = __this_cpu_read(irq_epoch);
+		info->eoi_time = get_jiffies_64() + event_eoi_delay;
+	}
+
+	generic_handle_irq(irq);
+}
+
 static void __xen_evtchn_do_upcall(void)
 {
 	struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
 	int cpu = smp_processor_id();
+	struct evtchn_loop_ctrl ctrl = { 0 };
 
 	read_lock(&evtchn_rwlock);
 
 	do {
 		vcpu_info->evtchn_upcall_pending = 0;
 
-		xen_evtchn_handle_events(cpu);
+		xen_evtchn_handle_events(cpu, &ctrl);
 
 		BUG_ON(!irqs_disabled());
 
 		virt_rmb(); /* Hypervisor can set upcall pending. */
 
 	} while (vcpu_info->evtchn_upcall_pending);
 
 	read_unlock(&evtchn_rwlock);
+
+	/*
+	 * Increment irq_epoch only now to defer EOIs only for
+	 * xen_irq_lateeoi() invocations occurring from inside the loop
+	 * above.
+	 */
+	__this_cpu_inc(irq_epoch);
 }
 
 void xen_evtchn_do_upcall(struct pt_regs *regs)
 {
 	struct pt_regs *old_regs = set_irq_regs(regs);
@@ -1823,20 +2001,19 @@ static __init void xen_alloc_callback_vector(void)
 #else
 void xen_setup_callback_vector(void) {}
 static inline void xen_alloc_callback_vector(void) {}
 #endif
 
-#undef MODULE_PARAM_PREFIX
-#define MODULE_PARAM_PREFIX "xen."
-
 static bool fifo_events = true;
 module_param(fifo_events, bool, 0);
 
 static int xen_evtchn_cpu_prepare(unsigned int cpu)
 {
 	int ret = 0;
 
+	xen_cpu_init_eoi(cpu);
+
 	if (evtchn_ops->percpu_init)
 		ret = evtchn_ops->percpu_init(cpu);
 
 	return ret;
 }
@@ -1859,10 +2036,12 @@ void __init xen_init_IRQ(void)
 	if (fifo_events)
 		ret = xen_evtchn_fifo_init();
 	if (ret < 0)
 		xen_evtchn_2l_init();
 
+	xen_cpu_init_eoi(smp_processor_id());
+
 	cpuhp_setup_state_nocalls(CPUHP_XEN_EVTCHN_PREPARE,
 				  "xen/evtchn:prepare",
 				  xen_evtchn_cpu_prepare, xen_evtchn_cpu_dead);
 
 	evtchn_to_irq = kcalloc(EVTCHN_ROW(xen_evtchn_max_channels()),
diff --git a/drivers/xen/events/events_fifo.c b/drivers/xen/events/events_fifo.c
index 40e4ca1685aa..6085a808da95 100644
--- a/drivers/xen/events/events_fifo.c
+++ b/drivers/xen/events/events_fifo.c
@@ -273,23 +273,13 @@ static uint32_t clear_linked(volatile event_word_t *word)
 	} while ((w = sync_cmpxchg(word, old, new)) != old);
 
 	return w & EVTCHN_FIFO_LINK_MASK;
 }
 
-static void handle_irq_for_port(evtchn_port_t port)
-{
-	int irq;
-
-	irq = get_evtchn_to_irq(port);
-	if (irq != -1)
-		generic_handle_irq(irq);
-}
-
-static void consume_one_event(unsigned cpu,
+static void consume_one_event(unsigned cpu, struct evtchn_loop_ctrl *ctrl,
 			      struct evtchn_fifo_control_block *control_block,
-			      unsigned priority, unsigned long *ready,
-			      bool drop)
+			      unsigned priority, unsigned long *ready)
 {
 	struct evtchn_fifo_queue *q = &per_cpu(cpu_queue, cpu);
 	uint32_t head;
 	evtchn_port_t port;
 	event_word_t *word;
@@ -318,20 +308,21 @@ static void consume_one_event(unsigned cpu,
 	 */
 	if (head == 0)
 		clear_bit(priority, ready);
 
 	if (evtchn_fifo_is_pending(port) && !evtchn_fifo_is_masked(port)) {
-		if (unlikely(drop))
+		if (unlikely(!ctrl))
 			pr_warn("Dropping pending event for port %u\n", port);
 		else
-			handle_irq_for_port(port);
+			handle_irq_for_port(port, ctrl);
 	}
 
 	q->head[priority] = head;
 }
 
-static void __evtchn_fifo_handle_events(unsigned cpu, bool drop)
+static void __evtchn_fifo_handle_events(unsigned cpu,
+					struct evtchn_loop_ctrl *ctrl)
 {
 	struct evtchn_fifo_control_block *control_block;
 	unsigned long ready;
 	unsigned q;
 
@@ -339,18 +330,19 @@ static void __evtchn_fifo_handle_events(unsigned cpu, bool drop)
 
 	ready = xchg(&control_block->ready, 0);
 
 	while (ready) {
 		q = find_first_bit(&ready, EVTCHN_FIFO_MAX_QUEUES);
-		consume_one_event(cpu, control_block, q, &ready, drop);
+		consume_one_event(cpu, ctrl, control_block, q, &ready);
 		ready |= xchg(&control_block->ready, 0);
 	}
 }
 
-static void evtchn_fifo_handle_events(unsigned cpu)
+static void evtchn_fifo_handle_events(unsigned cpu,
+				      struct evtchn_loop_ctrl *ctrl)
 {
-	__evtchn_fifo_handle_events(cpu, false);
+	__evtchn_fifo_handle_events(cpu, ctrl);
 }
 
 static void evtchn_fifo_resume(void)
 {
 	unsigned cpu;
@@ -414,11 +406,11 @@ static int evtchn_fifo_percpu_init(unsigned int cpu)
 	return 0;
 }
 
 static int evtchn_fifo_percpu_deinit(unsigned int cpu)
 {
-	__evtchn_fifo_handle_events(cpu, true);
+	__evtchn_fifo_handle_events(cpu, NULL);
 	return 0;
 }
 
 static const struct evtchn_ops evtchn_ops_fifo = {
 	.max_channels      = evtchn_fifo_max_channels,
diff --git a/drivers/xen/events/events_internal.h b/drivers/xen/events/events_internal.h
index 558abea19d0d..aac05cf52ced 100644
--- a/drivers/xen/events/events_internal.h
+++ b/drivers/xen/events/events_internal.h
@@ -28,15 +28,19 @@ enum xen_irq_type {
  *    IPI - IPI vector
  *    EVTCHN -
  */
 struct irq_info {
 	struct list_head list;
+	struct list_head eoi_list;
 	int refcnt;
 	enum xen_irq_type type;	/* type */
 	unsigned irq;
 	evtchn_port_t evtchn;	/* event channel */
 	unsigned short cpu;	/* cpu bound */
+	unsigned short eoi_cpu;	/* EOI must happen on this cpu */
+	unsigned int irq_epoch;	/* If eoi_cpu valid: irq_epoch of event */
+	u64 eoi_time;		/* Time in jiffies when to EOI. */
 
 	union {
 		unsigned short virq;
 		enum ipi_vector ipi;
 		struct {
@@ -51,10 +55,12 @@ struct irq_info {
 
 #define PIRQ_NEEDS_EOI	(1 << 0)
 #define PIRQ_SHAREABLE	(1 << 1)
 #define PIRQ_MSI_GROUP	(1 << 2)
 
+struct evtchn_loop_ctrl;
+
 struct evtchn_ops {
 	unsigned (*max_channels)(void);
 	unsigned (*nr_channels)(void);
 
 	int (*setup)(struct irq_info *info);
@@ -65,21 +71,22 @@ struct evtchn_ops {
 	bool (*is_pending)(evtchn_port_t port);
 	bool (*test_and_set_mask)(evtchn_port_t port);
 	void (*mask)(evtchn_port_t port);
 	void (*unmask)(evtchn_port_t port);
 
-	void (*handle_events)(unsigned cpu);
+	void (*handle_events)(unsigned cpu, struct evtchn_loop_ctrl *ctrl);
 	void (*resume)(void);
 
 	int (*percpu_init)(unsigned int cpu);
 	int (*percpu_deinit)(unsigned int cpu);
 };
 
 extern const struct evtchn_ops *evtchn_ops;
 
 extern int **evtchn_to_irq;
 int get_evtchn_to_irq(evtchn_port_t evtchn);
+void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl);
 
 struct irq_info *info_for_irq(unsigned irq);
 unsigned cpu_from_irq(unsigned irq);
 unsigned int cpu_from_evtchn(evtchn_port_t evtchn);
 
@@ -133,13 +140,14 @@ static inline void mask_evtchn(evtchn_port_t port)
 static inline void unmask_evtchn(evtchn_port_t port)
 {
 	return evtchn_ops->unmask(port);
 }
 
-static inline void xen_evtchn_handle_events(unsigned cpu)
+static inline void xen_evtchn_handle_events(unsigned cpu,
+					    struct evtchn_loop_ctrl *ctrl)
 {
-	return evtchn_ops->handle_events(cpu);
+	return evtchn_ops->handle_events(cpu, ctrl);
 }
 
 static inline void xen_evtchn_resume(void)
 {
 	if (evtchn_ops->resume)

Vulnerable code: static void evtchn_2l_handle_events(unsigned cpu)
			irq = get_evtchn_to_irq(port);

			if (irq != -1)
				generic_handle_irq(irq);
	if (!VALID_EVTCHN(evtchn))
		xen_evtchn_handle_events(cpu);
#undef MODULE_PARAM_PREFIX
#define MODULE_PARAM_PREFIX "xen."

static void handle_irq_for_port(evtchn_port_t port)
{
	int irq;

	irq = get_evtchn_to_irq(port);
	if (irq != -1)
		generic_handle_irq(irq);
}

static void consume_one_event(unsigned cpu,
			      unsigned priority, unsigned long *ready,
			      bool drop)
		if (unlikely(drop))
			handle_irq_for_port(port);
static void __evtchn_fifo_handle_events(unsigned cpu, bool drop)
		consume_one_event(cpu, control_block, q, &ready, drop);
static void evtchn_fifo_handle_events(unsigned cpu)
	__evtchn_fifo_handle_events(cpu, false);
	__evtchn_fifo_handle_events(cpu, true);
	void (*handle_events)(unsigned cpu);
static inline void xen_evtchn_handle_events(unsigned cpu)
	return evtchn_ops->handle_events(cpu);
Fixed code: 	xen.event_eoi_delay=	[XEN]
			How long to delay EOI handling in case of event
			storms (jiffies). Default is 10.

	xen.event_loop_timeout=	[XEN]
			After which time (jiffies) the event handling loop
			should start to delay EOI handling. Default is 2.

static void evtchn_2l_handle_events(unsigned cpu, struct evtchn_loop_ctrl *ctrl)
			handle_irq_for_port(port, ctrl);
#include <linux/atomic.h>
#include <linux/ktime.h>
#undef MODULE_PARAM_PREFIX
#define MODULE_PARAM_PREFIX "xen."

static uint __read_mostly event_loop_timeout = 2;
module_param(event_loop_timeout, uint, 0644);

static uint __read_mostly event_eoi_delay = 10;
module_param(event_eoi_delay, uint, 0644);

 *       percpu eoi_list_lock
static DEFINE_PER_CPU(unsigned int, irq_epoch);

struct lateeoi_work {
	struct delayed_work delayed;
	spinlock_t eoi_list_lock;
	struct list_head eoi_list;
};

static DEFINE_PER_CPU(struct lateeoi_work, lateeoi);

static void lateeoi_list_del(struct irq_info *info)
{
	struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
	unsigned long flags;

	spin_lock_irqsave(&eoi->eoi_list_lock, flags);
	list_del_init(&info->eoi_list);
	spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
}

static void lateeoi_list_add(struct irq_info *info)
{
	struct lateeoi_work *eoi = &per_cpu(lateeoi, info->eoi_cpu);
	struct irq_info *elem;
	u64 now = get_jiffies_64();
	unsigned long delay;
	unsigned long flags;

	if (now < info->eoi_time)
		delay = info->eoi_time - now;
	else
		delay = 1;

	spin_lock_irqsave(&eoi->eoi_list_lock, flags);

	if (list_empty(&eoi->eoi_list)) {
		list_add(&info->eoi_list, &eoi->eoi_list);
		mod_delayed_work_on(info->eoi_cpu, system_wq,
				    &eoi->delayed, delay);
	} else {
		list_for_each_entry_reverse(elem, &eoi->eoi_list, eoi_list) {
			if (elem->eoi_time <= info->eoi_time)
				break;
		}
		list_add(&info->eoi_list, &elem->eoi_list);
	}

	spin_unlock_irqrestore(&eoi->eoi_list_lock, flags);
}

	unsigned int cpu;
	if (!VALID_EVTCHN(evtchn) || !list_empty(&info->eoi_list))
	cpu = info->eoi_cpu;
	if (info->eoi_time && info->irq_epoch == per_cpu(irq_epoch, cpu)) {
		lateeoi_list_add(info);
		return;
	}

	info->eoi_time = 0;
static void xen_irq_lateeoi_worker(struct work_struct *work)
{
	struct lateeoi_work *eoi;
	struct irq_info *info;
	u64 now = get_jiffies_64();
	unsigned long flags;

	eoi = container_of(to_delayed_work(work), struct lateeoi_work, delayed);

	read_lock_irqsave(&evtchn_rwlock, flags);

	while (true) {
		spin_lock(&eoi->eoi_list_lock);

		info = list_first_entry_or_null(&eoi->eoi_list, struct irq_info,
						eoi_list);

		if (info == NULL || now < info->eoi_time) {
			spin_unlock(&eoi->eoi_list_lock);
			break;
		}

		list_del_init(&info->eoi_list);

		spin_unlock(&eoi->eoi_list_lock);

		info->eoi_time = 0;

		xen_irq_lateeoi_locked(info);
	}

	if (info)
		mod_delayed_work_on(info->eoi_cpu, system_wq,
				    &eoi->delayed, info->eoi_time - now);

	read_unlock_irqrestore(&evtchn_rwlock, flags);
}

static void xen_cpu_init_eoi(unsigned int cpu)
{
	struct lateeoi_work *eoi = &per_cpu(lateeoi, cpu);

	INIT_DELAYED_WORK(&eoi->delayed, xen_irq_lateeoi_worker);
	spin_lock_init(&eoi->eoi_list_lock);
	INIT_LIST_HEAD(&eoi->eoi_list);
}


	INIT_LIST_HEAD(&info->eoi_list);
	if (!list_empty(&info->eoi_list))
		lateeoi_list_del(info);

struct evtchn_loop_ctrl {
	ktime_t timeout;
	unsigned count;
	bool defer_eoi;
};

void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl)
{
	int irq;
	struct irq_info *info;

	irq = get_evtchn_to_irq(port);
	if (irq == -1)
		return;

	/*
	 * Check for timeout every 256 events.
	 * We are setting the timeout value only after the first 256
	 * events in order to not hurt the common case of few loop
	 * iterations. The 256 is basically an arbitrary value.
	 *
	 * In case we are hitting the timeout we need to defer all further
	 * EOIs in order to ensure to leave the event handling loop rather
	 * sooner than later.
	 */
	if (!ctrl->defer_eoi && !(++ctrl->count & 0xff)) {
		ktime_t kt = ktime_get();

		if (!ctrl->timeout) {
			kt = ktime_add_ms(kt,
					  jiffies_to_msecs(event_loop_timeout));
			ctrl->timeout = kt;
		} else if (kt > ctrl->timeout) {
			ctrl->defer_eoi = true;
		}
	}

	info = info_for_irq(irq);

	if (ctrl->defer_eoi) {
		info->eoi_cpu = smp_processor_id();
		info->irq_epoch = __this_cpu_read(irq_epoch);
		info->eoi_time = get_jiffies_64() + event_eoi_delay;
	}

	generic_handle_irq(irq);
}

	struct evtchn_loop_ctrl ctrl = { 0 };
		xen_evtchn_handle_events(cpu, &ctrl);

	/*
	 * Increment irq_epoch only now to defer EOIs only for
	 * xen_irq_lateeoi() invocations occurring from inside the loop
	 * above.
	 */
	__this_cpu_inc(irq_epoch);
	xen_cpu_init_eoi(cpu);

	xen_cpu_init_eoi(smp_processor_id());

static void consume_one_event(unsigned cpu, struct evtchn_loop_ctrl *ctrl,
			      unsigned priority, unsigned long *ready)
		if (unlikely(!ctrl))
			handle_irq_for_port(port, ctrl);
static void __evtchn_fifo_handle_events(unsigned cpu,
					struct evtchn_loop_ctrl *ctrl)
		consume_one_event(cpu, ctrl, control_block, q, &ready);
static void evtchn_fifo_handle_events(unsigned cpu,
				      struct evtchn_loop_ctrl *ctrl)
	__evtchn_fifo_handle_events(cpu, ctrl);
	__evtchn_fifo_handle_events(cpu, NULL);
	struct list_head eoi_list;
	unsigned short eoi_cpu;	/* EOI must happen on this cpu */
	unsigned int irq_epoch;	/* If eoi_cpu valid: irq_epoch of event */
	u64 eoi_time;		/* Time in jiffies when to EOI. */
struct evtchn_loop_ctrl;

	void (*handle_events)(unsigned cpu, struct evtchn_loop_ctrl *ctrl);
void handle_irq_for_port(evtchn_port_t port, struct evtchn_loop_ctrl *ctrl);
static inline void xen_evtchn_handle_events(unsigned cpu,
					    struct evtchn_loop_ctrl *ctrl)
	return evtchn_ops->handle_events(cpu, ctrl);