From 68a792174d7f67c7d2108bf1cc55ab8a63fc4678 Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Mon, 12 Jun 2017 16:41:41 -0400 Subject: sparc64: remove trailing white spaces A few changes that were reported by checkpatch, removed all trailing white spaces in these two files. Signed-off-by: Pavel Tatashin Reviewed-by: Shannon Nelson Signed-off-by: David S. Miller --- arch/sparc/include/asm/setup.h | 2 +- arch/sparc/kernel/setup_64.c | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/sparc/include/asm/setup.h b/arch/sparc/include/asm/setup.h index 3fae200dd251..8b32538084f7 100644 --- a/arch/sparc/include/asm/setup.h +++ b/arch/sparc/include/asm/setup.h @@ -1,5 +1,5 @@ /* - * Just a place holder. + * Just a place holder. */ #ifndef _SPARC_SETUP_H #define _SPARC_SETUP_H diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index 422b17880955..4a85f9241a2a 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -95,7 +95,7 @@ static struct console prom_early_console = { .index = -1, }; -/* +/* * Process kernel command line switches that are specific to the * SPARC or that require special low-level processing. */ @@ -639,7 +639,7 @@ void __init setup_arch(char **cmdline_p) #ifdef CONFIG_BLK_DEV_RAM rd_image_start = ram_flags & RAMDISK_IMAGE_START_MASK; rd_prompt = ((ram_flags & RAMDISK_PROMPT_FLAG) != 0); - rd_doload = ((ram_flags & RAMDISK_LOAD_FLAG) != 0); + rd_doload = ((ram_flags & RAMDISK_LOAD_FLAG) != 0); #endif task_thread_info(&init_task)->kregs = &fake_swapper_regs; @@ -648,7 +648,7 @@ void __init setup_arch(char **cmdline_p) if (!ic_set_manually) { phandle chosen = prom_finddevice("/chosen"); u32 cl, sv, gw; - + cl = prom_getintdefault (chosen, "client-ip", 0); sv = prom_getintdefault (chosen, "server-ip", 0); gw = prom_getintdefault (chosen, "gateway-ip", 0); -- cgit v1.2.3-55-g7522 From b8a83fcb78c859b99807af4c8b0ab09f0f827a40 Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Mon, 12 Jun 2017 16:41:42 -0400 Subject: sparc64: access tick function from variable In timer_64.c tick functions are access via pointer (tick_ops), every time clock is read, there is one extra load to get to the function. This patch optimizes it, by accessing functions pointer from value. Current ched_clock(): sethi %hi(0xb9b400), %g1 ldx [ %g1 + 0x250 ], %g1 ! ldx [ %g1 ], %g1 call %g1 nop sethi %hi(0xb9b400), %g1 ldx [ %g1 + 0x300 ], %g1 ! mulx %o0, %g1, %g1 rett %i7 + 8 srlx %g1, 0xa, %o0 New sched_clock(): sethi %hi(0xb9b400), %g1 ldx [ %g1 + 0x340 ], %g1 call %g1 nop sethi %hi(0xb9b400), %g1 ldx [ %g1 + 0x378 ], %g1 mulx %o0, %g1, %g1 rett %i7 + 8 srlx %g1, 0xa, %o0 Before three loads, now two loads. Signed-off-by: Pavel Tatashin Reviewed-by: Shannon Nelson Reviewed-by: Steven Sistare Signed-off-by: David S. Miller --- arch/sparc/kernel/time_64.c | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 98d05de8da66..6724bcbc3526 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -585,6 +585,7 @@ fs_initcall(clock_init); /* This is gets the master TICK_INT timer going. */ static unsigned long sparc64_init_timers(void) { + struct sparc64_tick_ops *ops = NULL; struct device_node *dp; unsigned long freq; @@ -598,16 +599,17 @@ static unsigned long sparc64_init_timers(void) impl = ((ver >> 32) & 0xffff); if (manuf == 0x17 && impl == 0x13) { /* Hummingbird, aka Ultra-IIe */ - tick_ops = &hbtick_operations; + ops = &hbtick_operations; freq = of_getintprop_default(dp, "stick-frequency", 0); } else { - tick_ops = &tick_operations; freq = local_cpu_data().clock_tick; } } else { - tick_ops = &stick_operations; + ops = &stick_operations; freq = of_getintprop_default(dp, "stick-frequency", 0); } + if (ops) + memcpy(&tick_operations, ops, sizeof(struct sparc64_tick_ops)); return freq; } @@ -671,12 +673,12 @@ core_initcall(register_sparc64_cpufreq_notifier); static int sparc64_next_event(unsigned long delta, struct clock_event_device *evt) { - return tick_ops->add_compare(delta) ? -ETIME : 0; + return tick_operations.add_compare(delta) ? -ETIME : 0; } static int sparc64_timer_shutdown(struct clock_event_device *evt) { - tick_ops->disable_irq(); + tick_operations.disable_irq(); return 0; } @@ -693,7 +695,7 @@ static DEFINE_PER_CPU(struct clock_event_device, sparc64_events); void __irq_entry timer_interrupt(int irq, struct pt_regs *regs) { struct pt_regs *old_regs = set_irq_regs(regs); - unsigned long tick_mask = tick_ops->softint_mask; + unsigned long tick_mask = tick_operations.softint_mask; int cpu = smp_processor_id(); struct clock_event_device *evt = &per_cpu(sparc64_events, cpu); @@ -728,7 +730,7 @@ void setup_sparc64_timer(void) : "=r" (pstate) : "i" (PSTATE_IE)); - tick_ops->init_tick(); + tick_operations.init_tick(); /* Restore PSTATE_IE. */ __asm__ __volatile__("wrpr %0, 0x0, %%pstate" @@ -757,9 +759,9 @@ void __delay(unsigned long loops) { unsigned long bclock, now; - bclock = tick_ops->get_tick(); + bclock = tick_operations.get_tick(); do { - now = tick_ops->get_tick(); + now = tick_operations.get_tick(); } while ((now-bclock) < loops); } EXPORT_SYMBOL(__delay); @@ -772,7 +774,7 @@ EXPORT_SYMBOL(udelay); static u64 clocksource_tick_read(struct clocksource *cs) { - return tick_ops->get_tick(); + return tick_operations.get_tick(); } void __init time_init(void) @@ -784,14 +786,14 @@ void __init time_init(void) timer_ticks_per_nsec_quotient = clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); - clocksource_tick.name = tick_ops->name; + clocksource_tick.name = tick_operations.name; clocksource_tick.read = clocksource_tick_read; clocksource_register_hz(&clocksource_tick, freq); printk("clocksource: mult[%x] shift[%d]\n", clocksource_tick.mult, clocksource_tick.shift); - sparc64_clockevent.name = tick_ops->name; + sparc64_clockevent.name = tick_operations.name; clockevents_calc_mult_shift(&sparc64_clockevent, freq, 4); sparc64_clockevent.max_delta_ns = @@ -809,7 +811,7 @@ void __init time_init(void) unsigned long long sched_clock(void) { - unsigned long ticks = tick_ops->get_tick(); + unsigned long ticks = tick_operations.get_tick(); return (ticks * timer_ticks_per_nsec_quotient) >> SPARC64_NSEC_PER_CYC_SHIFT; @@ -817,6 +819,6 @@ unsigned long long sched_clock(void) int read_current_timer(unsigned long *timer_val) { - *timer_val = tick_ops->get_tick(); + *timer_val = tick_operations.get_tick(); return 0; } -- cgit v1.2.3-55-g7522 From b5dd4d807f0fe7da67c5cc67b2ec681b60e4994b Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Mon, 12 Jun 2017 16:41:43 -0400 Subject: sparc64: show time stamps from zero On most platforms, time is shown from the beginning of boot. This patch is adding offset to sched_clock() for SPARC, to also show time from 0. This means we will have one more load, but we saved one in an ealier patch. Signed-off-by: Pavel Tatashin Reviewed-by: Bob Picco Reviewed-by: Steven Sistare Signed-off-by: David S. Miller --- arch/sparc/kernel/time_64.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 6724bcbc3526..5f53b74dd493 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -392,6 +392,7 @@ static struct sparc64_tick_ops hbtick_operations __read_mostly = { }; static unsigned long timer_ticks_per_nsec_quotient __read_mostly; +static unsigned long timer_offset __read_mostly; unsigned long cmos_regs; EXPORT_SYMBOL(cmos_regs); @@ -786,6 +787,10 @@ void __init time_init(void) timer_ticks_per_nsec_quotient = clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); + timer_offset = (tick_operations.get_tick() + * timer_ticks_per_nsec_quotient) + >> SPARC64_NSEC_PER_CYC_SHIFT; + clocksource_tick.name = tick_operations.name; clocksource_tick.read = clocksource_tick_read; @@ -813,8 +818,9 @@ unsigned long long sched_clock(void) { unsigned long ticks = tick_operations.get_tick(); - return (ticks * timer_ticks_per_nsec_quotient) - >> SPARC64_NSEC_PER_CYC_SHIFT; + return ((ticks * timer_ticks_per_nsec_quotient) + >> SPARC64_NSEC_PER_CYC_SHIFT) + - timer_offset; } int read_current_timer(unsigned long *timer_val) -- cgit v1.2.3-55-g7522 From 178bf2b9a20e866677bbca5cb521b09a8498c1d7 Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Mon, 12 Jun 2017 16:41:44 -0400 Subject: sparc64: optimize loads in clock_sched() In clock sched we now have three loads: - Function pointer - quotient for multiplication - offset However, it is possible to improve performance substantially, by guaranteeing that all three loads are from the same cacheline. By moving these three values first in sparc64_tick_ops, and by having tick_operations 64-byte aligned we guarantee this. Signed-off-by: Pavel Tatashin Reviewed-by: Shannon Nelson Reviewed-by: Steven Sistare Signed-off-by: David S. Miller --- arch/sparc/include/asm/timer_64.h | 5 +++++ arch/sparc/kernel/time_64.c | 17 +++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/arch/sparc/include/asm/timer_64.h b/arch/sparc/include/asm/timer_64.h index fce415034000..bde2cc40ae02 100644 --- a/arch/sparc/include/asm/timer_64.h +++ b/arch/sparc/include/asm/timer_64.h @@ -9,7 +9,12 @@ #include #include +/* The most frequently accessed fields should be first, + * to fit into the same cacheline. + */ struct sparc64_tick_ops { + unsigned long ticks_per_nsec_quotient; + unsigned long offset; unsigned long long (*get_tick)(void); int (*add_compare)(unsigned long); unsigned long softint_mask; diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 5f53b74dd493..44e37e9f8428 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -164,7 +164,7 @@ static unsigned long tick_add_tick(unsigned long adj) return new_tick; } -static struct sparc64_tick_ops tick_operations __read_mostly = { +static struct sparc64_tick_ops tick_operations __cacheline_aligned = { .name = "tick", .init_tick = tick_init_tick, .disable_irq = tick_disable_irq, @@ -391,9 +391,6 @@ static struct sparc64_tick_ops hbtick_operations __read_mostly = { .softint_mask = 1UL << 0, }; -static unsigned long timer_ticks_per_nsec_quotient __read_mostly; -static unsigned long timer_offset __read_mostly; - unsigned long cmos_regs; EXPORT_SYMBOL(cmos_regs); @@ -784,11 +781,11 @@ void __init time_init(void) tb_ticks_per_usec = freq / USEC_PER_SEC; - timer_ticks_per_nsec_quotient = + tick_operations.ticks_per_nsec_quotient = clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); - timer_offset = (tick_operations.get_tick() - * timer_ticks_per_nsec_quotient) + tick_operations.offset = (tick_operations.get_tick() + * tick_operations.ticks_per_nsec_quotient) >> SPARC64_NSEC_PER_CYC_SHIFT; clocksource_tick.name = tick_operations.name; @@ -816,11 +813,11 @@ void __init time_init(void) unsigned long long sched_clock(void) { + unsigned long quotient = tick_operations.ticks_per_nsec_quotient; + unsigned long offset = tick_operations.offset; unsigned long ticks = tick_operations.get_tick(); - return ((ticks * timer_ticks_per_nsec_quotient) - >> SPARC64_NSEC_PER_CYC_SHIFT) - - timer_offset; + return ((ticks * quotient) >> SPARC64_NSEC_PER_CYC_SHIFT) - offset; } int read_current_timer(unsigned long *timer_val) -- cgit v1.2.3-55-g7522 From 89108c3423e8047cd0da73182ea09b9da190b57e Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Mon, 12 Jun 2017 16:41:45 -0400 Subject: sparc64: improve modularity tick options This patch prepares the code for early boot time stamps by making it more modular. - init_tick_ops() to initialize struct sparc64_tick_ops - new sparc64_tick_ops operation get_frequency() which returns a frequency Signed-off-by: Pavel Tatashin Reviewed-by: Bob Picco Reviewed-by: Steven Sistare Signed-off-by: David S. Miller --- arch/sparc/include/asm/timer_64.h | 2 + arch/sparc/kernel/time_64.c | 83 ++++++++++++++++++++++++++------------- 2 files changed, 57 insertions(+), 28 deletions(-) diff --git a/arch/sparc/include/asm/timer_64.h b/arch/sparc/include/asm/timer_64.h index bde2cc40ae02..61d07251792c 100644 --- a/arch/sparc/include/asm/timer_64.h +++ b/arch/sparc/include/asm/timer_64.h @@ -22,6 +22,8 @@ struct sparc64_tick_ops { void (*init_tick)(void); unsigned long (*add_tick)(unsigned long); + unsigned long (*get_frequency)(void); + unsigned long frequency; char *name; }; diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 44e37e9f8428..3bd9e499755b 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -164,6 +164,11 @@ static unsigned long tick_add_tick(unsigned long adj) return new_tick; } +static unsigned long tick_get_frequency(void) +{ + return local_cpu_data().clock_tick; +} + static struct sparc64_tick_ops tick_operations __cacheline_aligned = { .name = "tick", .init_tick = tick_init_tick, @@ -171,6 +176,7 @@ static struct sparc64_tick_ops tick_operations __cacheline_aligned = { .get_tick = tick_get_tick, .add_tick = tick_add_tick, .add_compare = tick_add_compare, + .get_frequency = tick_get_frequency, .softint_mask = 1UL << 0, }; @@ -250,6 +256,13 @@ static int stick_add_compare(unsigned long adj) return ((long)(new_tick - (orig_tick+adj))) > 0L; } +static unsigned long stick_get_frequency(void) +{ + struct device_node *dp = of_find_node_by_path("/"); + + return of_getintprop_default(dp, "stick-frequency", 0); +} + static struct sparc64_tick_ops stick_operations __read_mostly = { .name = "stick", .init_tick = stick_init_tick, @@ -257,6 +270,7 @@ static struct sparc64_tick_ops stick_operations __read_mostly = { .get_tick = stick_get_tick, .add_tick = stick_add_tick, .add_compare = stick_add_compare, + .get_frequency = stick_get_frequency, .softint_mask = 1UL << 16, }; @@ -381,6 +395,13 @@ static int hbtick_add_compare(unsigned long adj) return ((long)(val2 - val)) > 0L; } +static unsigned long hbtick_get_frequency(void) +{ + struct device_node *dp = of_find_node_by_path("/"); + + return of_getintprop_default(dp, "stick-frequency", 0); +} + static struct sparc64_tick_ops hbtick_operations __read_mostly = { .name = "hbtick", .init_tick = hbtick_init_tick, @@ -388,6 +409,7 @@ static struct sparc64_tick_ops hbtick_operations __read_mostly = { .get_tick = hbtick_get_tick, .add_tick = hbtick_add_tick, .add_compare = hbtick_add_compare, + .get_frequency = hbtick_get_frequency, .softint_mask = 1UL << 0, }; @@ -580,36 +602,17 @@ static int __init clock_init(void) */ fs_initcall(clock_init); -/* This is gets the master TICK_INT timer going. */ -static unsigned long sparc64_init_timers(void) +/* Return true if this is Hummingbird, aka Ultra-IIe */ +static bool is_hummingbird(void) { - struct sparc64_tick_ops *ops = NULL; - struct device_node *dp; - unsigned long freq; + unsigned long ver, manuf, impl; - dp = of_find_node_by_path("/"); - if (tlb_type == spitfire) { - unsigned long ver, manuf, impl; - - __asm__ __volatile__ ("rdpr %%ver, %0" - : "=&r" (ver)); - manuf = ((ver >> 48) & 0xffff); - impl = ((ver >> 32) & 0xffff); - if (manuf == 0x17 && impl == 0x13) { - /* Hummingbird, aka Ultra-IIe */ - ops = &hbtick_operations; - freq = of_getintprop_default(dp, "stick-frequency", 0); - } else { - freq = local_cpu_data().clock_tick; - } - } else { - ops = &stick_operations; - freq = of_getintprop_default(dp, "stick-frequency", 0); - } - if (ops) - memcpy(&tick_operations, ops, sizeof(struct sparc64_tick_ops)); + __asm__ __volatile__ ("rdpr %%ver, %0" + : "=&r" (ver)); + manuf = ((ver >> 48) & 0xffff); + impl = ((ver >> 32) & 0xffff); - return freq; + return (manuf == 0x17 && impl == 0x13); } struct freq_table { @@ -775,10 +778,34 @@ static u64 clocksource_tick_read(struct clocksource *cs) return tick_operations.get_tick(); } +static void init_tick_ops(struct sparc64_tick_ops *ops) +{ + unsigned long freq, quotient, tick; + + freq = ops->get_frequency(); + quotient = clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); + tick = ops->get_tick(); + + ops->offset = (tick * quotient) >> SPARC64_NSEC_PER_CYC_SHIFT; + ops->ticks_per_nsec_quotient = quotient; + ops->frequency = freq; + tick_operations = *ops; +} + void __init time_init(void) { - unsigned long freq = sparc64_init_timers(); + unsigned long freq; + + if (tlb_type == spitfire) { + if (is_hummingbird()) + init_tick_ops(&hbtick_operations); + else + init_tick_ops(&tick_operations); + } else { + init_tick_ops(&stick_operations); + } + freq = tick_operations.frequency; tb_ticks_per_usec = freq / USEC_PER_SEC; tick_operations.ticks_per_nsec_quotient = -- cgit v1.2.3-55-g7522 From 83e8eb99d908da78e6eff7dd141f26626fe01d12 Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Mon, 12 Jun 2017 16:41:46 -0400 Subject: sparc64: initialize time early In Linux it is possible to configure printk() to output timestamp next to every line. This is very useful to determine the slow parts of the boot process, and also to avoid regressions, as boot time is visiable to everyone. Also, there are scripts that change these time stamps to intervals. However, on larger machines these timestamps start appearing many seconds, and even minutes into the boot process. This patch gets stick-frequency property early from OpenBoot, and uses its value to initialize time stamps before the first printk() messages are printed. Signed-off-by: Pavel Tatashin Reviewed-by: Shannon Nelson Reviewed-by: Steven Sistare Signed-off-by: David S. Miller --- arch/sparc/kernel/kernel.h | 3 +++ arch/sparc/kernel/setup_64.c | 1 + arch/sparc/kernel/time_64.c | 21 ++++++++------------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/arch/sparc/kernel/kernel.h b/arch/sparc/kernel/kernel.h index 6ae1e77be0bf..b625db4cfb78 100644 --- a/arch/sparc/kernel/kernel.h +++ b/arch/sparc/kernel/kernel.h @@ -52,6 +52,9 @@ asmlinkage void do_rt_sigreturn32(struct pt_regs *regs); void do_signal32(struct pt_regs * regs); asmlinkage int do_sys32_sigstack(u32 u_ssptr, u32 u_ossptr, unsigned long sp); +/* time_64.c */ +void __init time_init_early(void); + /* compat_audit.c */ extern unsigned int sparc32_dir_class[]; extern unsigned int sparc32_chattr_class[]; diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index 4a85f9241a2a..4d9c3e13c150 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -365,6 +365,7 @@ void __init start_early_boot(void) } current_thread_info()->cpu = cpu; + time_init_early(); prom_init_report(); start_kernel(); } diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index 3bd9e499755b..d149276ddd80 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -49,6 +49,7 @@ #include #include "entry.h" +#include "kernel.h" DEFINE_SPINLOCK(rtc_lock); @@ -258,9 +259,7 @@ static int stick_add_compare(unsigned long adj) static unsigned long stick_get_frequency(void) { - struct device_node *dp = of_find_node_by_path("/"); - - return of_getintprop_default(dp, "stick-frequency", 0); + return prom_getint(prom_root_node, "stick-frequency"); } static struct sparc64_tick_ops stick_operations __read_mostly = { @@ -792,10 +791,8 @@ static void init_tick_ops(struct sparc64_tick_ops *ops) tick_operations = *ops; } -void __init time_init(void) +void __init time_init_early(void) { - unsigned long freq; - if (tlb_type == spitfire) { if (is_hummingbird()) init_tick_ops(&hbtick_operations); @@ -804,17 +801,15 @@ void __init time_init(void) } else { init_tick_ops(&stick_operations); } +} + +void __init time_init(void) +{ + unsigned long freq; freq = tick_operations.frequency; tb_ticks_per_usec = freq / USEC_PER_SEC; - tick_operations.ticks_per_nsec_quotient = - clocksource_hz2mult(freq, SPARC64_NSEC_PER_CYC_SHIFT); - - tick_operations.offset = (tick_operations.get_tick() - * tick_operations.ticks_per_nsec_quotient) - >> SPARC64_NSEC_PER_CYC_SHIFT; - clocksource_tick.name = tick_operations.name; clocksource_tick.read = clocksource_tick_read; -- cgit v1.2.3-55-g7522 From 4929c83a6ce6584cb64381bf1407c487f67d588a Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Mon, 12 Jun 2017 16:41:47 -0400 Subject: sparc64: add hot-patched and inlined get_tick() Add the new get_tick() function that is hot-patched during boot based on processor we are booting on. Signed-off-by: Pavel Tatashin Reviewed-by: Steven Sistare Signed-off-by: David S. Miller --- arch/sparc/include/asm/timer_64.h | 60 +++++++++++++++++++++++++++++++++++++++ arch/sparc/kernel/time_64.c | 28 ++++++++++++++---- arch/sparc/kernel/vmlinux.lds.S | 5 ++++ 3 files changed, 87 insertions(+), 6 deletions(-) diff --git a/arch/sparc/include/asm/timer_64.h b/arch/sparc/include/asm/timer_64.h index 61d07251792c..51bc3bc54bfe 100644 --- a/arch/sparc/include/asm/timer_64.h +++ b/arch/sparc/include/asm/timer_64.h @@ -34,4 +34,64 @@ unsigned long sparc64_get_clock_tick(unsigned int cpu); void setup_sparc64_timer(void); void __init time_init(void); +#define TICK_PRIV_BIT BIT(63) +#define TICKCMP_IRQ_BIT BIT(63) + +#define HBIRD_STICKCMP_ADDR 0x1fe0000f060UL +#define HBIRD_STICK_ADDR 0x1fe0000f070UL + +#define GET_TICK_NINSTR 13 +struct get_tick_patch { + unsigned int addr; + unsigned int tick[GET_TICK_NINSTR]; + unsigned int stick[GET_TICK_NINSTR]; +}; + +extern struct get_tick_patch __get_tick_patch; +extern struct get_tick_patch __get_tick_patch_end; + +static inline unsigned long get_tick(void) +{ + unsigned long tick, tmp1, tmp2; + + __asm__ __volatile__( + /* read hbtick 13 instructions */ + "661:\n" + " mov 0x1fe, %1\n" + " sllx %1, 0x20, %1\n" + " sethi %%hi(0xf000), %2\n" + " or %2, 0x70, %2\n" + " or %1, %2, %1\n" /* %1 = HBIRD_STICK_ADDR */ + " add %1, 8, %2\n" + " ldxa [%2]%3, %0\n" + " ldxa [%1]%3, %1\n" + " ldxa [%2]%3, %2\n" + " sub %2, %0, %0\n" /* don't modify %xcc */ + " brnz,pn %0, 661b\n" /* restart to save one register */ + " sllx %2, 32, %2\n" + " or %2, %1, %0\n" + /* Common/not patched code */ + " sllx %0, 1, %0\n" + " srlx %0, 1, %0\n" /* Clear TICK_PRIV_BIT */ + /* Beginning of patch section */ + " .section .get_tick_patch, \"ax\"\n" + " .word 661b\n" + /* read tick 2 instructions and 11 skipped */ + " ba 1f\n" + " rd %%tick, %0\n" + " .skip 4 * (%4 - 2)\n" + "1:\n" + /* read stick 2 instructions and 11 skipped */ + " ba 1f\n" + " rd %%asr24, %0\n" + " .skip 4 * (%4 - 2)\n" + "1:\n" + /* End of patch section */ + " .previous\n" + : "=&r" (tick), "=&r" (tmp1), "=&r" (tmp2) + : "i" (ASI_PHYS_BYPASS_EC_E), "i" (GET_TICK_NINSTR)); + + return tick; +} + #endif /* _SPARC64_TIMER_H */ diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index d149276ddd80..ca27415c393a 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -47,15 +47,13 @@ #include #include #include +#include #include "entry.h" #include "kernel.h" DEFINE_SPINLOCK(rtc_lock); -#define TICK_PRIV_BIT (1UL << 63) -#define TICKCMP_IRQ_BIT (1UL << 63) - #ifdef CONFIG_SMP unsigned long profile_pc(struct pt_regs *regs) { @@ -290,9 +288,6 @@ static struct sparc64_tick_ops stick_operations __read_mostly = { * 2) write high * 3) write low */ -#define HBIRD_STICKCMP_ADDR 0x1fe0000f060UL -#define HBIRD_STICK_ADDR 0x1fe0000f070UL - static unsigned long __hbird_read_stick(void) { unsigned long ret, tmp1, tmp2, tmp3; @@ -777,6 +772,26 @@ static u64 clocksource_tick_read(struct clocksource *cs) return tick_operations.get_tick(); } +static void __init get_tick_patch(void) +{ + unsigned int *addr, *instr, i; + struct get_tick_patch *p; + + if (tlb_type == spitfire && is_hummingbird()) + return; + + for (p = &__get_tick_patch; p < &__get_tick_patch_end; p++) { + instr = (tlb_type == spitfire) ? p->tick : p->stick; + addr = (unsigned int *)(unsigned long)p->addr; + for (i = 0; i < GET_TICK_NINSTR; i++) { + addr[i] = instr[i]; + /* ensure that address is modified before flush */ + wmb(); + flushi(&addr[i]); + } + } +} + static void init_tick_ops(struct sparc64_tick_ops *ops) { unsigned long freq, quotient, tick; @@ -789,6 +804,7 @@ static void init_tick_ops(struct sparc64_tick_ops *ops) ops->ticks_per_nsec_quotient = quotient; ops->frequency = freq; tick_operations = *ops; + get_tick_patch(); } void __init time_init_early(void) diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 572db686f845..03b3d65d1266 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -149,6 +149,11 @@ SECTIONS *(.sun_m7_2insn_patch) __sun_m7_2insn_patch_end = .; } + .get_tick_patch : { + __get_tick_patch = .; + *(.get_tick_patch) + __get_tick_patch_end = .; + } PERCPU_SECTION(SMP_CACHE_BYTES) #ifdef CONFIG_JUMP_LABEL -- cgit v1.2.3-55-g7522 From eae3fc9871111e9bbc77dad5481a3e805e02ac46 Mon Sep 17 00:00:00 2001 From: Pavel Tatashin Date: Mon, 12 Jun 2017 16:41:48 -0400 Subject: sparc64: optimize functions that access tick Replace read tick function pointers with the new hot-patched get_tick(). This optimizes the performance of functions such as: sched_clock() Signed-off-by: Pavel Tatashin Reviewed-by: Steven Sistare Signed-off-by: David S. Miller --- arch/sparc/kernel/time_64.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/arch/sparc/kernel/time_64.c b/arch/sparc/kernel/time_64.c index ca27415c393a..a612a91cb9cd 100644 --- a/arch/sparc/kernel/time_64.c +++ b/arch/sparc/kernel/time_64.c @@ -752,12 +752,10 @@ static unsigned long tb_ticks_per_usec __read_mostly; void __delay(unsigned long loops) { - unsigned long bclock, now; + unsigned long bclock = get_tick(); - bclock = tick_operations.get_tick(); - do { - now = tick_operations.get_tick(); - } while ((now-bclock) < loops); + while ((get_tick() - bclock) < loops) + ; } EXPORT_SYMBOL(__delay); @@ -769,7 +767,7 @@ EXPORT_SYMBOL(udelay); static u64 clocksource_tick_read(struct clocksource *cs) { - return tick_operations.get_tick(); + return get_tick(); } static void __init get_tick_patch(void) @@ -853,13 +851,19 @@ unsigned long long sched_clock(void) { unsigned long quotient = tick_operations.ticks_per_nsec_quotient; unsigned long offset = tick_operations.offset; - unsigned long ticks = tick_operations.get_tick(); - return ((ticks * quotient) >> SPARC64_NSEC_PER_CYC_SHIFT) - offset; + /* Use barrier so the compiler emits the loads first and overlaps load + * latency with reading tick, because reading %tick/%stick is a + * post-sync instruction that will flush and restart subsequent + * instructions after it commits. + */ + barrier(); + + return ((get_tick() * quotient) >> SPARC64_NSEC_PER_CYC_SHIFT) - offset; } int read_current_timer(unsigned long *timer_val) { - *timer_val = tick_operations.get_tick(); + *timer_val = get_tick(); return 0; } -- cgit v1.2.3-55-g7522