[PATCH v2 3/4] x86, mwaitt: introduce mwaix delay with a configurable timer

[Huang Rui <ray.huang@amd.com>]

MWAITX can enable a timer and a corresponding timer value specified in SW
P0 clocks. The SW P0 frequency is the same with TSC. The timer provides an
upper bound on how long the instruction waits before exiting.

The implementation of delay function in kernel can lerverage the timer of
MWAITX. This patch provides a new method (delay_mwaitx) to measure delay
time.

Suggested-by: Andy Lutomirski <luto@amacapital.net>
Suggested-by: Borislav Petkov <bp@suse.de>
Signed-off-by: Huang Rui <ray.huang@amd.com>
---
 arch/x86/include/asm/delay.h |    1 +
 arch/x86/include/asm/mwait.h |   10 ++++++++++
 arch/x86/kernel/setup.c      |    3 +++
 arch/x86/lib/delay.c         |   34 +++++++++++++++++++++++++++++++++-
 4 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/arch/x86/include/asm/delay.h b/arch/x86/include/asm/delay.h
index 99873ec..ef9e411 100644
--- a/arch/x86/include/asm/delay.h
+++ b/arch/x86/include/asm/delay.h
@@ -10,6 +10,7 @@ extern unsigned long boot_option_delay;
 enum delay_type {
 	DELAY_LOOP=0,
 	DELAY_TSC,
+	DELAY_MWAITX,
 };
 
 #endif /* _ASM_X86_DELAY_H */
diff --git a/arch/x86/include/asm/mwait.h b/arch/x86/include/asm/mwait.h
index ece8048..9895119 100644
--- a/arch/x86/include/asm/mwait.h
+++ b/arch/x86/include/asm/mwait.h
@@ -14,6 +14,8 @@
 #define CPUID5_ECX_INTERRUPT_BREAK	0x2
 
 #define MWAIT_ECX_INTERRUPT_BREAK	0x1
+#define MWAITX_ECX_TIMER_ENABLE		BIT(1)
+#define MWAITX_MAX_LOOPS    		(u32)-1
 
 static inline void __monitor(const void *eax, unsigned long ecx,
 			     unsigned long edx)
@@ -80,4 +82,12 @@ static inline void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
 	current_clr_polling();
 }
 
+static inline void mwaitx(unsigned long loops, bool enable)
+{
+	if (enable)
+		__mwaitx(0, loops, MWAITX_ECX_TIMER_ENABLE);
+	else
+		__mwaitx(0, 0, 0);
+}
+
 #endif /* _ASM_X86_MWAIT_H */
diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c
index cc2886d..6b6f200 100644
--- a/arch/x86/kernel/setup.c
+++ b/arch/x86/kernel/setup.c
@@ -856,6 +856,9 @@ static int __init delay_setup(char *str)
 	} else if (!strcmp(str, "loop")) {
 		pr_info("using loop delay\n");
 		boot_option_delay = DELAY_LOOP;
+	} else if (!strcmp(str, "mwaitx")) {
+		pr_info("using mwaitx delay\n");
+		boot_option_delay = DELAY_MWAITX;
 	} else
 		return -1;
 
diff --git a/arch/x86/lib/delay.c b/arch/x86/lib/delay.c
index 1a6952e..2fb408b 100644
--- a/arch/x86/lib/delay.c
+++ b/arch/x86/lib/delay.c
@@ -20,6 +20,7 @@
 #include <asm/processor.h>
 #include <asm/delay.h>
 #include <asm/timer.h>
+#include <asm/mwait.h>
 
 #ifdef CONFIG_SMP
 # include <asm/smp.h>
@@ -89,6 +90,32 @@ static void delay_tsc(unsigned long __loops)
 }
 
 /*
+ * On AMD platforms mwaitx has a configurable 32-bit timer, that counts
+ * with TSC frequency. And the input value is the loop of the counter, it
+ * will exit with the timer expired.
+ */
+static void delay_mwaitx(unsigned long __loops)
+{
+	u32 end, now, delay, addr;
+
+	delay = __loops;
+	rdtsc_barrier();
+	rdtscl(end);
+	end += delay;
+
+	while (1) {
+		__monitorx(&addr, 0, 0);
+		mwaitx(delay, true);
+
+		rdtsc_barrier();
+		rdtscl(now);
+		if (end <= now)
+			break;
+		delay = end - now;
+	}
+}
+
+/*
  * Since we calibrate only once at boot, this
  * function should be set once at boot and not changed
  */
@@ -118,7 +145,12 @@ int read_current_timer(unsigned long *timer_val)
 
 void __delay(unsigned long loops)
 {
-	delay_fn(loops);
+	if (loops > MWAITX_MAX_LOOPS ||
+			!static_cpu_has_safe(X86_FEATURE_MWAITT) ||
+			boot_option_delay != DELAY_MWAITX)
+		delay_fn(loops);
+	else
+		delay_mwaitx(loops);
 }
 EXPORT_SYMBOL(__delay);
 
-- 
1.7.9.5