[PATCH 5/6] x86/irq: soft_moderation: add support for posted_msi (intel)

[Luigi Rizzo <lrizzo@google.com>]

On recent Intel CPUs, kernels compiled with CONFIG_X86_POSTED_MSI=y,
and the boot option "intremap=posted_msi", all MSI interrupts
that hit a CPU issue a single POSTED_MSI interrupt processed by
sysvec_posted_msi_notification() instead of having separate interrupts.

This change adds soft moderation hooks to the above handler.
Soft moderation on posted_msi does not require per-source enable,
irq_moderation.delay_us > 0 suffices.

To test it, run a kernel with the above options and enable moderation by
setting delay_us > 0.  The column "from_msi" in /proc/irq/soft_moderation
will show a non-zero value.

Change-Id: Idcd6005f05048c4b3f9d002c8587039b46bc9d73
---
 arch/x86/kernel/irq.c          | 12 +++++++
 include/linux/irq_moderation.h | 62 ++++++++++++++++++++++++++++++++++
 kernel/irq/irq_moderation.c    | 39 ++++++++++++++++-----
 3 files changed, 104 insertions(+), 9 deletions(-)

diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index 10721a1252269..241d57fadc30c 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -4,6 +4,7 @@
  */
 #include <linux/cpu.h>
 #include <linux/interrupt.h>
+#include <linux/irq_moderation.h>
 #include <linux/kernel_stat.h>
 #include <linux/of.h>
 #include <linux/seq_file.h>
@@ -448,6 +449,13 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification)
 	inc_irq_stat(posted_msi_notification_count);
 	irq_enter();
 
+	if (posted_msi_moderation_enabled()) {
+		if (posted_msi_should_rearm(handle_pending_pir(pid->pir, regs)))
+			goto rearm;
+		else
+			goto common_end;
+	}
+
 	/*
 	 * Max coalescing count includes the extra round of handle_pending_pir
 	 * after clearing the outstanding notification bit. Hence, at most
@@ -458,6 +466,7 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification)
 			break;
 	}
 
+rearm:
 	/*
 	 * Clear outstanding notification bit to allow new IRQ notifications,
 	 * do this last to maximize the window of interrupt coalescing.
@@ -471,6 +480,9 @@ DEFINE_IDTENTRY_SYSVEC(sysvec_posted_msi_notification)
 	 */
 	handle_pending_pir(pid->pir, regs);
 
+common_end:
+	posted_msi_moderation_epilogue();
+
 	apic_eoi();
 	irq_exit();
 	set_irq_regs(old_regs);
diff --git a/include/linux/irq_moderation.h b/include/linux/irq_moderation.h
index 45df60230e42e..aabcfaba1aefc 100644
--- a/include/linux/irq_moderation.h
+++ b/include/linux/irq_moderation.h
@@ -155,6 +155,14 @@ static inline void irq_moderation_hook(struct irq_desc *desc)
 	if (!irq_moderation_enabled())
 		return;
 
+#ifdef CONFIG_X86_POSTED_MSI
+	if (ms->in_posted_msi) { /* these calls are not moderated */
+		ms->from_posted_msi++;
+		ms->irq_count += irq_mod_info.count_msi_calls;
+		return;
+	}
+#endif
+
 	if (!READ_ONCE(desc->moderation_mode))
 		return;
 
@@ -193,11 +201,65 @@ static inline void irq_moderation_epilogue(const struct irq_desc *desc)
 		irq_moderation_start_timer(ms);
 }
 
+#ifdef CONFIG_X86_POSTED_MSI
+/*
+ * Helpers for to sysvec_posted_msi_notification(), use as follows
+ *
+ *	if (posted_msi_moderation_enabled()) {
+ *		if (posted_msi_should_rearm(handle_pending_pir(pid->pir, regs)))
+ *			goto rearm;
+ *		else
+ *			goto common_end;
+ *	}
+ *	...
+ *  common_end:
+ *	posted_msi_moderation_epilogue();
+ */
+static inline bool posted_msi_moderation_enabled(void)
+{
+	struct irq_mod_state *ms = this_cpu_ptr(&irq_mod_state);
+
+	if (!irq_moderation_enabled())
+		return false;
+	irq_moderation_adjust_delay(ms);
+	ms->in_posted_msi = true;	/* tell handler to not throttle these calls */
+	return true;
+}
+
+/* Decide whether to rearm or not posted_msi */
+static inline bool posted_msi_should_rearm(bool work_done)
+{
+	struct irq_mod_state *ms = this_cpu_ptr(&irq_mod_state);
+
+	if (ms->rounds_left > 0)	/* timer pending, no rearm */
+		return false;
+	if (!work_done)			/* no work done, can rearm */
+		return true;
+	if (!irq_moderation_needed(ms)) /* moderation not needed, rearm */
+		return true;
+	ms->kick_posted_msi = true;	/* do kick in timer callback */
+	irq_moderation_start_timer(ms);
+	return false;	/* timer now active, no rearm */
+}
+
+/* Cleanup state set in posted_msi_moderation_enabled() */
+static inline void posted_msi_moderation_epilogue(void)
+{
+	this_cpu_ptr(&irq_mod_state)->in_posted_msi = false;
+}
+#endif
+
 #else	/* empty stubs to avoid conditional compilation */
 
 static inline void irq_moderation_hook(struct irq_desc *desc) {}
 static inline void irq_moderation_epilogue(const struct irq_desc *desc) {}
 
+#ifdef CONFIG_X86_POSTED_MSI
+static inline bool posted_msi_moderation_enabled(void) { return false; }
+static inline bool posted_msi_should_rearm(bool work_done) { return false; }
+static inline void posted_msi_moderation_epilogue(void) {}
+#endif
+
 #endif /* CONFIG_IRQ_SOFT_MODERATION */
 
 #endif /* _LINUX_IRQ_MODERATION_H */
diff --git a/kernel/irq/irq_moderation.c b/kernel/irq/irq_moderation.c
index 0229697a6a95a..672e161ecf29e 100644
--- a/kernel/irq/irq_moderation.c
+++ b/kernel/irq/irq_moderation.c
@@ -7,6 +7,15 @@
 #include <linux/irq_moderation.h>
 #include <linux/kernel_stat.h>	/* interrupt.h, kcpustat_this_cpu */
 #include "internals.h"
+#ifdef CONFIG_X86
+#include <asm/apic.h>
+#endif
+
+#ifdef CONFIG_IRQ_REMAP
+extern bool enable_posted_msi;
+#else
+static bool enable_posted_msi;
+#endif
 
 /*
  * Platform-wide software interrupt moderation.
@@ -29,6 +38,10 @@
  *   moderation on that CPU/irq. If so, calls disable_irq_nosync() and starts
  *   an hrtimer with appropriate delay.
  *
+ * - Intel only: using "intremap=posted_msi", all the above is done in
+ *   sysvec_posted_msi_notification(). In this case all host device interrupts
+ *   are subject to moderation.
+ *
  * - the timer callback calls enable_irq() for all disabled interrupts on that
  *   CPU. That in turn will generate interrupts if there are pending events.
  *
@@ -82,6 +95,7 @@ struct irq_mod_info irq_mod_info ____cacheline_aligned = {
 	.delay_us = 100,
 	.update_ms = 1,
 	.scale_cpus = 100,
+	.count_msi_calls = true,
 	.count_timer_calls = true,
 	.decay_factor = 16,
 	.grow_factor = 8,
@@ -216,6 +230,17 @@ static enum hrtimer_restart moderation_timer_cb(struct hrtimer *timer)
 
 	ms->rounds_left--;
 
+#ifdef CONFIG_X86_POSTED_MSI
+	if (ms->kick_posted_msi) {
+		if (ms->rounds_left <= 0)
+			ms->kick_posted_msi = false;
+		/* Next call will be from timer, count it conditionally */
+		ms->irq_count -= !irq_mod_info.count_timer_calls;
+		ms->timer_calls++;
+		apic->send_IPI_self(POSTED_MSI_NOTIFICATION_VECTOR);
+	}
+#endif
+
 	if (ms->rounds_left > 0) {
 		/* Timer still alive. Just call the handlers */
 		list_for_each_entry_safe(desc, next, &ms->descs, ms_node) {
@@ -277,13 +302,6 @@ static void set_moderation_mode(struct irq_desc *desc, bool mode)
 	}
 }
 
-/* irq_to_desc is not exported. Wrap it in this function for a specific use. */
-void irq_moderation_set_mode(int irq, bool mode)
-{
-	set_moderation_mode(irq_to_desc(irq), mode);
-}
-EXPORT_SYMBOL(irq_moderation_set_mode);
-
 #pragma clang diagnostic error "-Wformat"
 /* Print statistics */
 static int moderation_show(struct seq_file *p, void *v)
@@ -325,7 +343,7 @@ static int moderation_show(struct seq_file *p, void *v)
 	}
 
 	seq_printf(p, "\n"
-		   "enabled              %s\n"
+		   "enabled              %s%s\n"
 		   "delay_us             %u\n"
 		   "target_irq_rate      %u\n"
 		   "hardirq_percent      %u\n"
@@ -333,13 +351,15 @@ static int moderation_show(struct seq_file *p, void *v)
 		   "update_ms            %u\n"
 		   "scale_cpus           %u\n"
 		   "count_timer_calls    %s\n"
+		   "count_msi_calls      %s\n"
 		   "decay_factor         %u\n"
 		   "grow_factor          %u\n",
-		   str_yes_no(delay_us),
+		   str_yes_no(delay_us), enable_posted_msi ? " (also on posted_msi)" : "",
 		   delay_us, irq_mod_info.target_irq_rate, irq_mod_info.hardirq_percent,
 		   irq_mod_info.timer_rounds, irq_mod_info.update_ms,
 		   irq_mod_info.scale_cpus,
 		   str_yes_no(irq_mod_info.count_timer_calls),
+		   str_yes_no(irq_mod_info.count_msi_calls),
 		   get_decay_factor(), get_grow_factor());
 
 	seq_printf(p,
@@ -375,6 +395,7 @@ struct var_names {
 	{ "timer_rounds", &irq_mod_info.timer_rounds, 0, 50 },
 	{ "update_ms", &irq_mod_info.update_ms, 1, 100 },
 	{ "count_timer_calls", &irq_mod_info.count_timer_calls, 0, 1 },
+	{ "count_msi_calls", &irq_mod_info.count_msi_calls, 0, 1 },
 	{}
 };
 
-- 
2.51.2.1041.gc1ab5b90ca-goog