[time] Allow timer to be selected at runtime

Allow the active timer (providing udelay() and currticks()) to be
selected at runtime based on probing during the INIT_EARLY stage of
initialisation.

TICKS_PER_SEC is now a fixed compile-time constant for all builds, and
is independent of the underlying clock tick rate.  We choose the value
1024 to allow multiplications and divisions on seconds to be converted
to bit shifts.

TICKS_PER_MS is defined as 1, allowing multiplications and divisions
on milliseconds to be omitted entirely.  The 2% inaccuracy in this
definition is negligible when using the standard BIOS timer (running
at around 18.2Hz).

TIMER_RDTSC now checks for a constant TSC before claiming to be a
usable timer.  (This timer can be tested in KVM via the command-line
option "-cpu host,+invtsc".)

Signed-off-by: Michael Brown <mcb30@ipxe.org>

9 files changed, 157 insertions, 156 deletions

diff --git a/src/arch/arm/include/bits/timer.h b/src/arch/arm/include/bits/timer.h
deleted file mode 100644
index 64e7d31d..00000000
--- a/src/arch/arm/include/bits/timer.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef _BITS_TIMER_H
-#define _BITS_TIMER_H
-
-/** @file
- *
- * ARM-specific timer API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#endif /* _BITS_TIMER_H */
diff --git a/src/arch/x86/core/rdtsc_timer.c b/src/arch/x86/core/rdtsc_timer.c
index e720a239..ed515150 100644
--- a/src/arch/x86/core/rdtsc_timer.c
+++ b/src/arch/x86/core/rdtsc_timer.c
@@ -29,16 +29,70 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  *
  */
 
-#include <assert.h>
+#include <string.h>
+#include <errno.h>
 #include <ipxe/timer.h>
+#include <ipxe/cpuid.h>
 #include <ipxe/pit8254.h>
 
+/** Number of microseconds to use for TSC calibration */
+#define TSC_CALIBRATE_US 1024
+
+/** TSC increment per microsecond */
+static unsigned long tsc_per_us;
+
+/** Minimum resolution for scaled TSC timer */
+#define TSC_SCALED_HZ 32
+
+/** TSC scale (expressed as a bit shift)
+ *
+ * We use this to avoid the need for 64-bit divsion on 32-bit systems.
+ */
+static unsigned int tsc_scale;
+
+/** Number of timer ticks per scaled TSC increment */
+static unsigned long ticks_per_scaled_tsc;
+
+/** Colour for debug messages */
+#define colour &tsc_per_us
+
+/**
+ * Get raw TSC value
+ *
+ * @ret tsc		Raw TSC value
+ */
+static inline __always_inline unsigned long rdtsc_raw ( void ) {
+	unsigned long raw;
+
+	__asm__ __volatile__ ( "rdtsc\n\t" : "=a" ( raw ) : : "edx" );
+	return raw;
+}
+
+/**
+ * Get TSC value, shifted to avoid rollover within a realistic timescale
+ *
+ * @ret tsc		Scaled TSC value
+ */
+static inline __always_inline unsigned long rdtsc_scaled ( void ) {
+	unsigned long scaled;
+
+	__asm__ __volatile__ ( "rdtsc\n\t"
+			       "shrdl %b1, %%edx, %%eax\n\t"
+			       : "=a" ( scaled ) : "c" ( tsc_scale ) : "edx" );
+	return scaled;
+}
+
 /**
- * Number of TSC ticks per microsecond
+ * Get current system time in ticks
  *
- * This is calibrated on the first use of the timer.
+ * @ret ticks		Current time, in ticks
  */
-static unsigned long rdtsc_ticks_per_usec;
+static unsigned long rdtsc_currticks ( void ) {
+	unsigned long scaled;
+
+	scaled = rdtsc_scaled();
+	return ( scaled * ticks_per_scaled_tsc );
+}
 
 /**
  * Delay for a fixed number of microseconds
@@ -48,47 +102,76 @@ static unsigned long rdtsc_ticks_per_usec;
 static void rdtsc_udelay ( unsigned long usecs ) {
 	unsigned long start;
 	unsigned long elapsed;
+	unsigned long threshold;
 
-	/* Sanity guard, since we may divide by this */
-	if ( ! usecs )
-		usecs = 1;
-
-	start = currticks();
-	if ( rdtsc_ticks_per_usec ) {
-		/* Already calibrated; busy-wait until done */
-		do {
-			elapsed = ( currticks() - start );
-		} while ( elapsed < ( usecs * rdtsc_ticks_per_usec ) );
-	} else {
-		/* Not yet calibrated; use 8254 PIT and calibrate
-		 * based on result.
-		 */
-		pit8254_udelay ( usecs );
-		elapsed = ( currticks() - start );
-		rdtsc_ticks_per_usec = ( elapsed / usecs );
-		DBG ( "RDTSC timer calibrated: %ld ticks in %ld usecs "
-		      "(%ld MHz)\n", elapsed, usecs,
-		      ( rdtsc_ticks_per_usec << TSC_SHIFT ) );
-	}
+	start = rdtsc_raw();
+	threshold = ( usecs * tsc_per_us );
+	do {
+		elapsed = ( rdtsc_raw() - start );
+	} while ( elapsed < threshold );
 }
 
 /**
- * Get number of ticks per second
+ * Probe RDTSC timer
  *
- * @ret ticks_per_sec	Number of ticks per second
+ * @ret rc		Return status code
  */
-static unsigned long rdtsc_ticks_per_sec ( void ) {
+static int rdtsc_probe ( void ) {
+	unsigned long before;
+	unsigned long after;
+	unsigned long elapsed;
+	uint32_t apm;
+	uint32_t discard_a;
+	uint32_t discard_b;
+	uint32_t discard_c;
+	int rc;
 
-	/* Calibrate timer, if not already done */
-	if ( ! rdtsc_ticks_per_usec )
-		udelay ( 1 );
+	/* Check that TSC is invariant */
+	if ( ( rc = cpuid_supported ( CPUID_APM ) ) != 0 ) {
+		DBGC ( colour, "RDTSC cannot determine APM features: %s\n",
+		       strerror ( rc ) );
+		return rc;
+	}
+	cpuid ( CPUID_APM, &discard_a, &discard_b, &discard_c, &apm );
+	if ( ! ( apm & CPUID_APM_EDX_TSC_INVARIANT ) ) {
+		DBGC ( colour, "RDTSC has non-invariant TSC (%#08x)\n",
+		       apm );
+		return -ENOTTY;
+	}
 
-	/* Sanity check */
-	assert ( rdtsc_ticks_per_usec != 0 );
+	/* Calibrate udelay() timer via 8254 PIT */
+	before = rdtsc_raw();
+	pit8254_udelay ( TSC_CALIBRATE_US );
+	after = rdtsc_raw();
+	elapsed = ( after - before );
+	tsc_per_us = ( elapsed / TSC_CALIBRATE_US );
+	if ( ! tsc_per_us ) {
+		DBGC ( colour, "RDTSC has zero TSC per microsecond\n" );
+		return -EIO;
+	}
+
+	/* Calibrate currticks() scaling factor */
+	tsc_scale = 31;
+	ticks_per_scaled_tsc = ( ( 1UL << tsc_scale ) /
+				 ( tsc_per_us * ( 1000000 / TICKS_PER_SEC ) ) );
+	while ( ticks_per_scaled_tsc > ( TICKS_PER_SEC / TSC_SCALED_HZ ) ) {
+		tsc_scale--;
+		ticks_per_scaled_tsc >>= 1;
+	}
+	DBGC ( colour, "RDTSC has %ld tsc per us, %ld ticks per 2^%d tsc\n",
+	       tsc_per_us, ticks_per_scaled_tsc, tsc_scale );
+	if ( ! ticks_per_scaled_tsc ) {
+		DBGC ( colour, "RDTSC has zero ticks per TSC\n" );
+		return -EIO;
+	}
 
-	return ( rdtsc_ticks_per_usec * 1000 * 1000 );
+	return 0;
 }
 
-PROVIDE_TIMER ( rdtsc, udelay, rdtsc_udelay );
-PROVIDE_TIMER_INLINE ( rdtsc, currticks );
-PROVIDE_TIMER ( rdtsc, ticks_per_sec, rdtsc_ticks_per_sec );
+/** RDTSC timer */
+struct timer rdtsc_timer __timer ( TIMER_PREFERRED ) = {
+	.name = "rdtsc",
+	.probe = rdtsc_probe,
+	.currticks = rdtsc_currticks,
+	.udelay = rdtsc_udelay,
+};
diff --git a/src/arch/x86/include/bios.h b/src/arch/x86/include/bios.h
index a5a5d887..14e7acbc 100644
--- a/src/arch/x86/include/bios.h
+++ b/src/arch/x86/include/bios.h
@@ -7,6 +7,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define BDA_EBDA 0x000e
 #define BDA_EQUIPMENT_WORD 0x0010
 #define BDA_FBMS 0x0013
+#define BDA_TICKS 0x006c
+#define BDA_MIDNIGHT 0x0070
 #define BDA_REBOOT 0x0072
 #define BDA_REBOOT_WARM 0x1234
 #define BDA_NUM_DRIVES 0x0075
diff --git a/src/arch/x86/include/bits/errfile.h b/src/arch/x86/include/bits/errfile.h
index 105cdf5d..9d1fed7f 100644
--- a/src/arch/x86/include/bits/errfile.h
+++ b/src/arch/x86/include/bits/errfile.h
@@ -26,6 +26,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #define ERRFILE_rtc_entropy	( ERRFILE_ARCH | ERRFILE_CORE | 0x000f0000 )
 #define ERRFILE_acpipwr		( ERRFILE_ARCH | ERRFILE_CORE | 0x00100000 )
 #define ERRFILE_cpuid		( ERRFILE_ARCH | ERRFILE_CORE | 0x00110000 )
+#define ERRFILE_rdtsc_timer	( ERRFILE_ARCH | ERRFILE_CORE | 0x00120000 )
 
 #define ERRFILE_bootsector     ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x00000000 )
 #define ERRFILE_bzimage	       ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x00010000 )
diff --git a/src/arch/x86/include/bits/timer.h b/src/arch/x86/include/bits/timer.h
deleted file mode 100644
index b0ff5ee1..00000000
--- a/src/arch/x86/include/bits/timer.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef _BITS_TIMER_H
-#define _BITS_TIMER_H
-
-/** @file
- *
- * x86-specific timer API implementations
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#include <ipxe/bios_timer.h>
-#include <ipxe/rdtsc_timer.h>
-
-#endif /* _BITS_TIMER_H */
diff --git a/src/arch/x86/include/ipxe/bios_timer.h b/src/arch/x86/include/ipxe/bios_timer.h
deleted file mode 100644
index 6b88a623..00000000
--- a/src/arch/x86/include/ipxe/bios_timer.h
+++ /dev/null
@@ -1,44 +0,0 @@
-#ifndef _IPXE_BIOS_TIMER_H
-#define _IPXE_BIOS_TIMER_H
-
-/** @file
- *
- * BIOS timer
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#ifdef TIMER_PCBIOS
-#define TIMER_PREFIX_pcbios
-#else
-#define TIMER_PREFIX_pcbios __pcbios_
-#endif
-
-#include <ipxe/pit8254.h>
-
-/**
- * Delay for a fixed number of microseconds
- *
- * @v usecs		Number of microseconds for which to delay
- */
-static inline __always_inline void
-TIMER_INLINE ( pcbios, udelay ) ( unsigned long usecs ) {
-	/* BIOS timer is not high-resolution enough for udelay(), so
-	 * we use the 8254 Programmable Interval Timer.
-	 */
-	pit8254_udelay ( usecs );
-}
-
-/**
- * Get number of ticks per second
- *
- * @ret ticks_per_sec	Number of ticks per second
- */
-static inline __always_inline unsigned long
-TIMER_INLINE ( pcbios, ticks_per_sec ) ( void ) {
-	/* BIOS timer ticks over at 18.2 ticks per second */
-	return 18;
-}
-
-#endif /* _IPXE_BIOS_TIMER_H */
diff --git a/src/arch/x86/include/ipxe/cpuid.h b/src/arch/x86/include/ipxe/cpuid.h
index 2e2cc7c1..a9df9f0d 100644
--- a/src/arch/x86/include/ipxe/cpuid.h
+++ b/src/arch/x86/include/ipxe/cpuid.h
@@ -57,6 +57,12 @@ struct x86_features {
 /** Get CPU model */
 #define CPUID_MODEL 0x80000002UL
 
+/** Get APM information */
+#define CPUID_APM 0x80000007UL
+
+/** Invariant TSC */
+#define CPUID_APM_EDX_TSC_INVARIANT 0x00000100UL
+
 /**
  * Issue CPUID instruction
  *
diff --git a/src/arch/x86/include/ipxe/rdtsc_timer.h b/src/arch/x86/include/ipxe/rdtsc_timer.h
deleted file mode 100644
index 598f4bb0..00000000
--- a/src/arch/x86/include/ipxe/rdtsc_timer.h
+++ /dev/null
@@ -1,39 +0,0 @@
-#ifndef _IPXE_RDTSC_TIMER_H
-#define _IPXE_RDTSC_TIMER_H
-
-/** @file
- *
- * RDTSC timer
- *
- */
-
-FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
-
-#ifdef TIMER_RDTSC
-#define TIMER_PREFIX_rdtsc
-#else
-#define TIMER_PREFIX_rdtsc __rdtsc_
-#endif
-
-/**
- * RDTSC values can easily overflow an unsigned long.  We discard the
- * low-order bits in order to obtain sensibly-scaled values.
- */
-#define TSC_SHIFT 8
-
-/**
- * Get current system time in ticks
- *
- * @ret ticks		Current time, in ticks
- */
-static inline __always_inline unsigned long
-TIMER_INLINE ( rdtsc, currticks ) ( void ) {
-	unsigned long ticks;
-
-	__asm__ __volatile__ ( "rdtsc\n\t"
-			       "shrdl %1, %%edx, %%eax\n\t"
-			       : "=a" ( ticks ) : "i" ( TSC_SHIFT ) : "edx" );
-	return ticks;
-}
-
-#endif /* _IPXE_RDTSC_TIMER_H */
diff --git a/src/arch/x86/interface/pcbios/bios_timer.c b/src/arch/x86/interface/pcbios/bios_timer.c
index 3299c9aa..49e1d226 100644
--- a/src/arch/x86/interface/pcbios/bios_timer.c
+++ b/src/arch/x86/interface/pcbios/bios_timer.c
@@ -32,6 +32,18 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/timer.h>
 #include <realmode.h>
 #include <bios.h>
+#include <ipxe/pit8254.h>
+
+/** Number of ticks per day
+ *
+ * This seems to be the normative value, as used by e.g. SeaBIOS to
+ * decide when to set the midnight rollover flag.
+ */
+#define BIOS_TICKS_PER_DAY 0x1800b0
+
+/** Number of ticks per BIOS tick */
+#define TICKS_PER_BIOS_TICK \
+	( ( TICKS_PER_SEC * 60 * 60 * 24 ) / BIOS_TICKS_PER_DAY )
 
 /**
  * Get current system time in ticks
@@ -43,7 +55,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
  * of calling timeofday BIOS interrupt.
  */
 static unsigned long bios_currticks ( void ) {
-	static int days = 0;
+	static uint32_t offset;
 	uint32_t ticks;
 	uint8_t midnight;
 
@@ -53,18 +65,25 @@ static unsigned long bios_currticks ( void ) {
 			       "nop\n\t"
 			       "cli\n\t" );
 
-	get_real ( ticks, BDA_SEG, 0x006c );
-	get_real ( midnight, BDA_SEG, 0x0070 );
+	/* Read current BIOS time of day */
+	get_real ( ticks, BDA_SEG, BDA_TICKS );
+	get_real ( midnight, BDA_SEG, BDA_MIDNIGHT );
 
+	/* Handle midnight rollover */
 	if ( midnight ) {
 		midnight = 0;
-		put_real ( midnight, BDA_SEG, 0x0070 );
-		days += 0x1800b0;
+		put_real ( midnight, BDA_SEG, BDA_MIDNIGHT );
+		offset += BIOS_TICKS_PER_DAY;
 	}
+	ticks += offset;
 
-	return ( days + ticks );
+	/* Convert to timer ticks */
+	return ( ticks * TICKS_PER_BIOS_TICK );
 }
 
-PROVIDE_TIMER_INLINE ( pcbios, udelay );
-PROVIDE_TIMER ( pcbios, currticks, bios_currticks );
-PROVIDE_TIMER_INLINE ( pcbios, ticks_per_sec );
+/** BIOS timer */
+struct timer bios_timer __timer ( TIMER_NORMAL ) = {
+	.name = "bios",
+	.currticks = bios_currticks,
+	.udelay = pit8254_udelay,
+};