From 2f0778afac79bd8d226225556858a636931eeabc Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Thu, 15 Dec 2011 12:19:23 +0100 Subject: ARM: 7205/2: sched_clock: allow sched_clock to be selected at runtime sched_clock() is yet another blocker on the road to the single image. This patch implements an idea by Russell King: http://www.spinics.net/lists/linux-omap/msg49561.html Instead of asking the platform to implement both sched_clock() itself and the rollover callback, simply register a read() function, and let the ARM code care about sched_clock() itself, the conversion to ns and the rollover. sched_clock() uses this read() function as an indirection to the platform code. If the platform doesn't provide a read(), the code falls back to the jiffy counter (just like the default sched_clock). This allow some simplifications and possibly some footprint gain when multiple platforms are compiled in. Among the drawbacks, the removal of the *_fixed_sched_clock optimization which could negatively impact some platforms (sa1100, tegra, versatile and omap). Tested on 11MPCore, OMAP4 and Tegra. Cc: Imre Kaloz Cc: Eric Miao Cc: Colin Cross Cc: Erik Gilling Cc: Olof Johansson Cc: Sascha Hauer Cc: Alessandro Rubini Cc: STEricsson Cc: Lennert Buytenhek Cc: Ben Dooks Tested-by: Jamie Iles Tested-by: Tony Lindgren Tested-by: Kyungmin Park Acked-by: Linus Walleij Acked-by: Nicolas Pitre Acked-by: Krzysztof Halasa Acked-by: Kukjin Kim Signed-off-by: Marc Zyngier Signed-off-by: Russell King --- arch/arm/kernel/sched_clock.c | 118 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 105 insertions(+), 13 deletions(-) (limited to 'arch/arm/kernel') diff --git a/arch/arm/kernel/sched_clock.c b/arch/arm/kernel/sched_clock.c index 9a46370fe9da..5416c7c12528 100644 --- a/arch/arm/kernel/sched_clock.c +++ b/arch/arm/kernel/sched_clock.c @@ -14,61 +14,153 @@ #include +struct clock_data { + u64 epoch_ns; + u32 epoch_cyc; + u32 epoch_cyc_copy; + u32 mult; + u32 shift; +}; + static void sched_clock_poll(unsigned long wrap_ticks); static DEFINE_TIMER(sched_clock_timer, sched_clock_poll, 0, 0); -static void (*sched_clock_update_fn)(void); + +static struct clock_data cd = { + .mult = NSEC_PER_SEC / HZ, +}; + +static u32 __read_mostly sched_clock_mask = 0xffffffff; + +static u32 notrace jiffy_sched_clock_read(void) +{ + return (u32)(jiffies - INITIAL_JIFFIES); +} + +static u32 __read_mostly (*read_sched_clock)(void) = jiffy_sched_clock_read; + +static inline u64 cyc_to_ns(u64 cyc, u32 mult, u32 shift) +{ + return (cyc * mult) >> shift; +} + +static unsigned long long cyc_to_sched_clock(u32 cyc, u32 mask) +{ + u64 epoch_ns; + u32 epoch_cyc; + + /* + * Load the epoch_cyc and epoch_ns atomically. We do this by + * ensuring that we always write epoch_cyc, epoch_ns and + * epoch_cyc_copy in strict order, and read them in strict order. + * If epoch_cyc and epoch_cyc_copy are not equal, then we're in + * the middle of an update, and we should repeat the load. + */ + do { + epoch_cyc = cd.epoch_cyc; + smp_rmb(); + epoch_ns = cd.epoch_ns; + smp_rmb(); + } while (epoch_cyc != cd.epoch_cyc_copy); + + return epoch_ns + cyc_to_ns((cyc - epoch_cyc) & mask, cd.mult, cd.shift); +} + +/* + * Atomically update the sched_clock epoch. + */ +static void notrace update_sched_clock(void) +{ + unsigned long flags; + u32 cyc; + u64 ns; + + cyc = read_sched_clock(); + ns = cd.epoch_ns + + cyc_to_ns((cyc - cd.epoch_cyc) & sched_clock_mask, + cd.mult, cd.shift); + /* + * Write epoch_cyc and epoch_ns in a way that the update is + * detectable in cyc_to_fixed_sched_clock(). + */ + raw_local_irq_save(flags); + cd.epoch_cyc = cyc; + smp_wmb(); + cd.epoch_ns = ns; + smp_wmb(); + cd.epoch_cyc_copy = cyc; + raw_local_irq_restore(flags); +} static void sched_clock_poll(unsigned long wrap_ticks) { mod_timer(&sched_clock_timer, round_jiffies(jiffies + wrap_ticks)); - sched_clock_update_fn(); + update_sched_clock(); } -void __init init_sched_clock(struct clock_data *cd, void (*update)(void), - unsigned int clock_bits, unsigned long rate) +void __init setup_sched_clock(u32 (*read)(void), int bits, unsigned long rate) { unsigned long r, w; u64 res, wrap; char r_unit; - sched_clock_update_fn = update; + BUG_ON(bits > 32); + WARN_ON(!irqs_disabled()); + WARN_ON(read_sched_clock != jiffy_sched_clock_read); + read_sched_clock = read; + sched_clock_mask = (1 << bits) - 1; /* calculate the mult/shift to convert counter ticks to ns. */ - clocks_calc_mult_shift(&cd->mult, &cd->shift, rate, NSEC_PER_SEC, 0); + clocks_calc_mult_shift(&cd.mult, &cd.shift, rate, NSEC_PER_SEC, 0); r = rate; if (r >= 4000000) { r /= 1000000; r_unit = 'M'; - } else { + } else if (r >= 1000) { r /= 1000; r_unit = 'k'; - } + } else + r_unit = ' '; /* calculate how many ns until we wrap */ - wrap = cyc_to_ns((1ULL << clock_bits) - 1, cd->mult, cd->shift); + wrap = cyc_to_ns((1ULL << bits) - 1, cd.mult, cd.shift); do_div(wrap, NSEC_PER_MSEC); w = wrap; /* calculate the ns resolution of this counter */ - res = cyc_to_ns(1ULL, cd->mult, cd->shift); + res = cyc_to_ns(1ULL, cd.mult, cd.shift); pr_info("sched_clock: %u bits at %lu%cHz, resolution %lluns, wraps every %lums\n", - clock_bits, r, r_unit, res, w); + bits, r, r_unit, res, w); /* * Start the timer to keep sched_clock() properly updated and * sets the initial epoch. */ sched_clock_timer.data = msecs_to_jiffies(w - (w / 10)); - update(); + update_sched_clock(); /* * Ensure that sched_clock() starts off at 0ns */ - cd->epoch_ns = 0; + cd.epoch_ns = 0; + + pr_debug("Registered %pF as sched_clock source\n", read); +} + +unsigned long long notrace sched_clock(void) +{ + u32 cyc = read_sched_clock(); + return cyc_to_sched_clock(cyc, sched_clock_mask); } void __init sched_clock_postinit(void) { + /* + * If no sched_clock function has been provided at that point, + * make it the final one one. + */ + if (read_sched_clock == jiffy_sched_clock_read) + setup_sched_clock(jiffy_sched_clock_read, 32, HZ); + sched_clock_poll(sched_clock_timer.data); } -- cgit v1.2.3-55-g7522