[PATCH RFC] x86, tsc: Allow for high latency in quick_pit_calibrate()

[Adrian Hunter <adrian.hunter@intel.com>]

If it takes longer than 12us to read the PIT counter lsb/msb,
then the error margin will never fall below 500ppm within 50ms,
and Fast TSC calibration will always fail.

This patch detects when that will happen and switches to using
a slightly different algorithm that takes advantage of the PIT's
latch comand.

Signed-off-by: Adrian Hunter <adrian.hunter@intel.com>
---
 arch/x86/kernel/tsc.c | 133 +++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 132 insertions(+), 1 deletion(-)

diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
index 5054497..0035682 100644
--- a/arch/x86/kernel/tsc.c
+++ b/arch/x86/kernel/tsc.c
@@ -553,6 +553,40 @@ static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *de
 }
 
 /*
+ * High latency version of pit_expect_msb(). Instead of using the read latency
+ * as the error margin, latch the counter and use that latency. The counter
+ * latches at the current value which means there is an addition error margin
+ * of 1 timer tick which is not accounted for here.
+ */
+static inline int pit_expect_msb_hl(unsigned char val, u64 *tscp,
+				    unsigned long *deltap, u16 *cnt)
+{
+	u64 tsc0, tsc1;
+	int count;
+	u8 lsb, msb;
+
+	for (count = 0; count < 50000; count++) {
+		tsc0 = get_cycles();
+		/* Latch counter 2 */
+		outb(0x80, 0x43);
+		tsc1 = get_cycles();
+		lsb = inb(0x42);
+		msb = inb(0x42);
+		if (msb != val)
+			break;
+	}
+	*deltap = tsc1 - tsc0;
+	*tscp = tsc0;
+	*cnt = lsb | ((u16)msb << 8);
+
+	/*
+	 * We require _some_ success, but the quality control
+	 * will be based on the error terms on the TSC values.
+	 */
+	return count > 5;
+}
+
+/*
  * How many MSB values do we want to see? We aim for
  * a maximum error rate of 500ppm (in practice the
  * real error is much smaller), but refuse to spend
@@ -561,6 +595,94 @@ static inline int pit_expect_msb(unsigned char val, u64 *tscp, unsigned long *de
 #define MAX_QUICK_PIT_MS 50
 #define MAX_QUICK_PIT_ITERATIONS (MAX_QUICK_PIT_MS * PIT_TICK_RATE / 1000 / 256)
 
+/*
+ * High latency version of quick_pit_calibrate() that works even if there is a
+ * high latency reading the PIT lsb/msb. This is needed if it takes longer than
+ * 12us to read the lsb/msb because then the error margin will never fall below
+ * 500ppm within 50ms.
+ */
+static unsigned long quick_pit_calibrate_hl(void)
+{
+	int i;
+	u64 tsc, delta;
+	unsigned long d1, d2;
+	u16 cnt0, cnt1, dc;
+
+	/* Re-start at 0xffff */
+	outb(0xff, 0x42);
+	outb(0xff, 0x42);
+
+	/*
+	 * The PIT starts counting at the next edge, so we
+	 * need to delay for a microsecond. The easiest way
+	 * to do that is to just read back the 16-bit counter
+	 * once from the PIT.
+	 */
+	pit_verify_msb(0);
+
+	/*
+	 * Iterate until the error is less than 500 ppm. The error margin due to
+	 * the time to latch the counter is d1 + d2. The counter latches the
+	 * current count which introduces a one tick error at the start and end.
+	 * So the total error is  d1 + d2 + 2 timer ticks. A timer tick is
+	 * approximately the TSC delta divided by the timer delta. So the error
+	 * margin is too high while:
+	 *      d1 + d2 + 2 * (delta / dc) >= delta >> 11
+	 *   => d1 + d2 >= (delta >> 11) - 2 * (delta / dc)
+	 *   => d1 + d2 >= (delta - 4096 * (delta / dc)) / 2048
+	 *   => (d1 + d2) * dc >= (dc * delta - 4096 * delta) / 2048
+	 *   => (d1 + d2) * dc >= (dc - 4096) * delta >> 11
+	*/
+	if (pit_expect_msb_hl(0xff, &tsc, &d1, &cnt0)) {
+		for (i = 1; i <= MAX_QUICK_PIT_ITERATIONS; i++) {
+			if (!pit_expect_msb_hl(0xff - i, &delta, &d2, &cnt1))
+				break;
+
+			dc = cnt0 - cnt1;
+			if (dc < 4096)
+				continue;
+
+			delta -= tsc;
+
+			if ((d1 + d2) * (u64)dc >= (dc - 4096) * delta >> 11)
+				continue;
+
+			/*
+			 * Check the PIT one more time to verify that
+			 * all TSC reads were stable wrt the PIT.
+			 *
+			 * This also guarantees serialization of the
+			 * last cycle read ('d2') in pit_expect_msb.
+			 */
+			if (!pit_verify_msb(0xfe - i))
+				break;
+			goto success;
+		}
+	}
+	pr_info("Fast TSC calibration (with latch) failed\n");
+	return 0;
+
+success:
+	/*
+	 * Ok, if we get here, then we've seen the
+	 * MSB of the PIT decrement 'i' times, and the
+	 * error has shrunk to less than 500 ppm.
+	 *
+	 * As a result, we can depend on there not being
+	 * any odd delays anywhere, and the TSC reads are
+	 * reliable (within the error).
+	 *
+	 * kHz = ticks / time-in-seconds / 1000;
+	 * kHz = ticks / (PIT count / PIT_TICK_RATE) / 1000
+	 * kHz = delta / (dc / PIT_TICK_RATE) / 1000
+	 * kHz = (delta * PIT_TICK_RATE) / (dc * 1000)
+	 */
+	delta *= PIT_TICK_RATE;
+	do_div(delta, dc * 1000);
+	pr_info("Fast TSC calibration (with latch) using PIT\n");
+	return delta;
+}
+
 static unsigned long quick_pit_calibrate(void)
 {
 	int i;
@@ -598,10 +720,19 @@ static unsigned long quick_pit_calibrate(void)
 			if (!pit_expect_msb(0xff-i, &delta, &d2))
 				break;
 
+			delta -= tsc;
+
+			/*
+			 * Extrapolate the error and switch to high-latency
+			 * algorithm if the error will never be below 500 ppm.
+			 */
+			if (i == 1 &&
+			    d1 + d2 >= (delta * MAX_QUICK_PIT_ITERATIONS) >> 11)
+				return quick_pit_calibrate_hl();
+
 			/*
 			 * Iterate until the error is less than 500 ppm
 			 */
-			delta -= tsc;
 			if (d1+d2 >= delta >> 11)
 				continue;
 
-- 
1.9.1