[time] Allow timer to be selected at runtime

Allow the active timer (providing udelay() and currticks()) to be
selected at runtime based on probing during the INIT_EARLY stage of
initialisation.

TICKS_PER_SEC is now a fixed compile-time constant for all builds, and
is independent of the underlying clock tick rate.  We choose the value
1024 to allow multiplications and divisions on seconds to be converted
to bit shifts.

TICKS_PER_MS is defined as 1, allowing multiplications and divisions
on milliseconds to be omitted entirely.  The 2% inaccuracy in this
definition is negligible when using the standard BIOS timer (running
at around 18.2Hz).

TIMER_RDTSC now checks for a constant TSC before claiming to be a
usable timer.  (This timer can be tested in KVM via the command-line
option "-cpu host,+invtsc".)

Signed-off-by: Michael Brown <mcb30@ipxe.org>

1 files changed, 120 insertions, 37 deletions

diff --git a/src/arch/x86/core/rdtsc_timer.c b/src/arch/x86/core/rdtsc_timer.c
index e720a239..ed515150 100644
--- a/src/arch/x86/core/rdtsc_timer.c
+++ b/src/arch/x86/core/rdtsc_timer.c
@@ -29,16 +29,70 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
-#include <assert.h>
+#include <string.h>
+#include <errno.h>
 #include <ipxe/timer.h>
+#include <ipxe/cpuid.h>
 #include <ipxe/pit8254.h>
 
+/** Number of microseconds to use for TSC calibration */
+#define TSC_CALIBRATE_US 1024
+
+/** TSC increment per microsecond */
+static unsigned long tsc_per_us;
+
+/** Minimum resolution for scaled TSC timer */
+#define TSC_SCALED_HZ 32
+
+/** TSC scale (expressed as a bit shift)
+ *
+ * We use this to avoid the need for 64-bit divsion on 32-bit systems.
+ */
+static unsigned int tsc_scale;
+
+/** Number of timer ticks per scaled TSC increment */
+static unsigned long ticks_per_scaled_tsc;
+
+/** Colour for debug messages */
+#define colour &tsc_per_us
+
+/**
+ * Get raw TSC value
+ *
+ * @ret tsc		Raw TSC value
+ */
+static inline __always_inline unsigned long rdtsc_raw ( void ) {
+	unsigned long raw;
+
+	__asm__ __volatile__ ( "rdtsc\n\t" : "=a" ( raw ) : : "edx" );
+	return raw;
+}
+
+/**
+ * Get TSC value, shifted to avoid rollover within a realistic timescale
+ *
+ * @ret tsc		Scaled TSC value
+ */
+static inline __always_inline unsigned long rdtsc_scaled ( void ) {
+	unsigned long scaled;
+
+	__asm__ __volatile__ ( "rdtsc\n\t"
+			       "shrdl %b1, %%edx, %%eax\n\t"
+			       : "=a" ( scaled ) : "c" ( tsc_scale ) : "edx" );
+	return scaled;
+}
+
 /**
- * Number of TSC ticks per microsecond
+ * Get current system time in ticks
  *
- * This is calibrated on the first use of the timer.
+ * @ret ticks		Current time, in ticks
  */
-static unsigned long rdtsc_ticks_per_usec;
+static unsigned long rdtsc_currticks ( void ) {
+	unsigned long scaled;
+
+	scaled = rdtsc_scaled();
+	return ( scaled * ticks_per_scaled_tsc );
+}
 
 /**
  * Delay for a fixed number of microseconds
@@ -48,47 +102,76 @@ static unsigned long rdtsc_ticks_per_usec;
 static void rdtsc_udelay ( unsigned long usecs ) {
 	unsigned long start;
 	unsigned long elapsed;
+	unsigned long threshold;
 
-	/* Sanity guard, since we may divide by this */
-	if ( ! usecs )
-		usecs = 1;
-
-	start = currticks();
-	if ( rdtsc_ticks_per_usec ) {
-		/* Already calibrated; busy-wait until done */
-		do {
-			elapsed = ( currticks() - start );
-		} while ( elapsed < ( usecs * rdtsc_ticks_per_usec ) );
-	} else {
-		/* Not yet calibrated; use 8254 PIT and calibrate
-		 * based on result.
-		 */
-		pit8254_udelay ( usecs );
-		elapsed = ( currticks() - start );
-		rdtsc_ticks_per_usec = ( elapsed / usecs );
-		DBG ( "RDTSC timer calibrated: %ld ticks in %ld usecs "
-		      "(%ld MHz)\n", elapsed, usecs,
-		      ( rdtsc_ticks_per_usec << TSC_SHIFT ) );
-	}
+	start = rdtsc_raw();
+	threshold = ( usecs * tsc_per_us );
+	do {
+		elapsed = ( rdtsc_raw() - start );
+	} while ( elapsed < threshold );
 }
 
 /**
- * Get number of ticks per second
+ * Probe RDTSC timer
  *
- * @ret ticks_per_sec	Number of ticks per second
+ * @ret rc		Return status code
  */
-static unsigned long rdtsc_ticks_per_sec ( void ) {
+static int rdtsc_probe ( void ) {
+	unsigned long before;
+	unsigned long after;
+	unsigned long elapsed;
+	uint32_t apm;
+	uint32_t discard_a;
+	uint32_t discard_b;
+	uint32_t discard_c;
+	int rc;
 
-	/* Calibrate timer, if not already done */
-	if ( ! rdtsc_ticks_per_usec )
-		udelay ( 1 );
+	/* Check that TSC is invariant */
+	if ( ( rc = cpuid_supported ( CPUID_APM ) ) != 0 ) {
+		DBGC ( colour, "RDTSC cannot determine APM features: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+	cpuid ( CPUID_APM, &discard_a, &discard_b, &discard_c, &apm );
+	if ( ! ( apm & CPUID_APM_EDX_TSC_INVARIANT ) ) {
+		DBGC ( colour, "RDTSC has non-invariant TSC (%#08x)\n",
+		       apm );
+		return -ENOTTY;
+	}
 
-	/* Sanity check */
-	assert ( rdtsc_ticks_per_usec != 0 );
+	/* Calibrate udelay() timer via 8254 PIT */
+	before = rdtsc_raw();
+	pit8254_udelay ( TSC_CALIBRATE_US );
+	after = rdtsc_raw();
+	elapsed = ( after - before );
+	tsc_per_us = ( elapsed / TSC_CALIBRATE_US );
+	if ( ! tsc_per_us ) {
+		DBGC ( colour, "RDTSC has zero TSC per microsecond\n" );
+		return -EIO;
+	}
+
+	/* Calibrate currticks() scaling factor */
+	tsc_scale = 31;
+	ticks_per_scaled_tsc = ( ( 1UL << tsc_scale ) /
+				 ( tsc_per_us * ( 1000000 / TICKS_PER_SEC ) ) );
+	while ( ticks_per_scaled_tsc > ( TICKS_PER_SEC / TSC_SCALED_HZ ) ) {
+		tsc_scale--;
+		ticks_per_scaled_tsc >>= 1;
+	}
+	DBGC ( colour, "RDTSC has %ld tsc per us, %ld ticks per 2^%d tsc\n",
+	       tsc_per_us, ticks_per_scaled_tsc, tsc_scale );
+	if ( ! ticks_per_scaled_tsc ) {
+		DBGC ( colour, "RDTSC has zero ticks per TSC\n" );
+		return -EIO;
+	}
 
-	return ( rdtsc_ticks_per_usec * 1000 * 1000 );
+	return 0;
 }
 
-PROVIDE_TIMER ( rdtsc, udelay, rdtsc_udelay );
-PROVIDE_TIMER_INLINE ( rdtsc, currticks );
-PROVIDE_TIMER ( rdtsc, ticks_per_sec, rdtsc_ticks_per_sec );
+/** RDTSC timer */
+struct timer rdtsc_timer __timer ( TIMER_PREFERRED ) = {
+	.name = "rdtsc",
+	.probe = rdtsc_probe,
+	.currticks = rdtsc_currticks,
+	.udelay = rdtsc_udelay,
+};