diff options
Diffstat (limited to 'arch/arm64/mm')
| -rw-r--r-- | arch/arm64/mm/cache.S | 21 | ||||
| -rw-r--r-- | arch/arm64/mm/fault.c | 236 | ||||
| -rw-r--r-- | arch/arm64/mm/proc.S | 22 |
3 files changed, 137 insertions, 142 deletions
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index 758bde7e2fa6..30334d81b021 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -50,6 +50,10 @@ ENTRY(flush_icache_range) */ ENTRY(__flush_cache_user_range) uaccess_ttbr0_enable x2, x3, x4 +alternative_if ARM64_HAS_CACHE_IDC + dsb ishst + b 7f +alternative_else_nop_endif dcache_line_size x2, x3 sub x3, x2, #1 bic x4, x0, x3 @@ -60,8 +64,13 @@ user_alt 9f, "dc cvau, x4", "dc civac, x4", ARM64_WORKAROUND_CLEAN_CACHE b.lo 1b dsb ish +7: +alternative_if ARM64_HAS_CACHE_DIC + isb + b 8f +alternative_else_nop_endif invalidate_icache_by_line x0, x1, x2, x3, 9f - mov x0, #0 +8: mov x0, #0 1: uaccess_ttbr0_disable x1, x2 ret @@ -80,6 +89,12 @@ ENDPROC(__flush_cache_user_range) * - end - virtual end address of region */ ENTRY(invalidate_icache_range) +alternative_if ARM64_HAS_CACHE_DIC + mov x0, xzr + isb + ret +alternative_else_nop_endif + uaccess_ttbr0_enable x2, x3, x4 invalidate_icache_by_line x0, x1, x2, x3, 2f @@ -116,6 +131,10 @@ ENDPIPROC(__flush_dcache_area) * - size - size in question */ ENTRY(__clean_dcache_area_pou) +alternative_if ARM64_HAS_CACHE_IDC + dsb ishst + ret +alternative_else_nop_endif dcache_by_line_op cvau, ish, x0, x1, x2, x3 ret ENDPROC(__clean_dcache_area_pou) diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index bff11553eb05..4165485e8b6e 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -43,6 +43,7 @@ #include <asm/system_misc.h> #include <asm/pgtable.h> #include <asm/tlbflush.h> +#include <asm/traps.h> #include <acpi/ghes.h> @@ -289,58 +290,31 @@ static void __do_kernel_fault(unsigned long addr, unsigned int esr, do_exit(SIGKILL); } -static void __do_user_fault(struct task_struct *tsk, unsigned long addr, - unsigned int esr, unsigned int sig, int code, - struct pt_regs *regs, int fault) +static void __do_user_fault(struct siginfo *info, unsigned int esr) { - struct siginfo si; - const struct fault_info *inf; - unsigned int lsb = 0; - - if (unhandled_signal(tsk, sig) && show_unhandled_signals_ratelimited()) { - inf = esr_to_fault_info(esr); - pr_info("%s[%d]: unhandled %s (%d) at 0x%08lx, esr 0x%03x", - tsk->comm, task_pid_nr(tsk), inf->name, sig, - addr, esr); - print_vma_addr(KERN_CONT ", in ", regs->pc); - pr_cont("\n"); - __show_regs(regs); - } - - tsk->thread.fault_address = addr; - tsk->thread.fault_code = esr; - si.si_signo = sig; - si.si_errno = 0; - si.si_code = code; - si.si_addr = (void __user *)addr; - /* - * Either small page or large page may be poisoned. - * In other words, VM_FAULT_HWPOISON_LARGE and - * VM_FAULT_HWPOISON are mutually exclusive. - */ - if (fault & VM_FAULT_HWPOISON_LARGE) - lsb = hstate_index_to_shift(VM_FAULT_GET_HINDEX(fault)); - else if (fault & VM_FAULT_HWPOISON) - lsb = PAGE_SHIFT; - si.si_addr_lsb = lsb; - - force_sig_info(sig, &si, tsk); + current->thread.fault_address = (unsigned long)info->si_addr; + current->thread.fault_code = esr; + arm64_force_sig_info(info, esr_to_fault_info(esr)->name, current); } static void do_bad_area(unsigned long addr, unsigned int esr, struct pt_regs *regs) { - struct task_struct *tsk = current; - const struct fault_info *inf; - /* * If we are in kernel mode at this point, we have no context to * handle this fault with. */ if (user_mode(regs)) { - inf = esr_to_fault_info(esr); - __do_user_fault(tsk, addr, esr, inf->sig, inf->code, regs, 0); - } else + const struct fault_info *inf = esr_to_fault_info(esr); + struct siginfo si = { + .si_signo = inf->sig, + .si_code = inf->code, + .si_addr = (void __user *)addr, + }; + + __do_user_fault(&si, esr); + } else { __do_kernel_fault(addr, esr, regs); + } } #define VM_FAULT_BADMAP 0x010000 @@ -393,7 +367,8 @@ static int __kprobes do_page_fault(unsigned long addr, unsigned int esr, { struct task_struct *tsk; struct mm_struct *mm; - int fault, sig, code, major = 0; + struct siginfo si; + int fault, major = 0; unsigned long vm_flags = VM_READ | VM_WRITE; unsigned int mm_flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE; @@ -525,27 +500,37 @@ retry: return 0; } + clear_siginfo(&si); + si.si_addr = (void __user *)addr; + if (fault & VM_FAULT_SIGBUS) { /* * We had some memory, but were unable to successfully fix up * this page fault. */ - sig = SIGBUS; - code = BUS_ADRERR; - } else if (fault & (VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE)) { - sig = SIGBUS; - code = BUS_MCEERR_AR; + si.si_signo = SIGBUS; + si.si_code = BUS_ADRERR; + } else if (fault & VM_FAULT_HWPOISON_LARGE) { + unsigned int hindex = VM_FAULT_GET_HINDEX(fault); + + si.si_signo = SIGBUS; + si.si_code = BUS_MCEERR_AR; + si.si_addr_lsb = hstate_index_to_shift(hindex); + } else if (fault & VM_FAULT_HWPOISON) { + si.si_signo = SIGBUS; + si.si_code = BUS_MCEERR_AR; + si.si_addr_lsb = PAGE_SHIFT; } else { /* * Something tried to access memory that isn't in our memory * map. */ - sig = SIGSEGV; - code = fault == VM_FAULT_BADACCESS ? - SEGV_ACCERR : SEGV_MAPERR; + si.si_signo = SIGSEGV; + si.si_code = fault == VM_FAULT_BADACCESS ? + SEGV_ACCERR : SEGV_MAPERR; } - __do_user_fault(tsk, addr, esr, sig, code, regs, fault); + __do_user_fault(&si, esr); return 0; no_context: @@ -582,8 +567,6 @@ static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs) const struct fault_info *inf; inf = esr_to_fault_info(esr); - pr_err("Synchronous External Abort: %s (0x%08x) at 0x%016lx\n", - inf->name, esr, addr); /* * Synchronous aborts may interrupt code which had interrupts masked. @@ -600,83 +583,83 @@ static int do_sea(unsigned long addr, unsigned int esr, struct pt_regs *regs) nmi_exit(); } - info.si_signo = SIGBUS; + info.si_signo = inf->sig; info.si_errno = 0; - info.si_code = BUS_FIXME; + info.si_code = inf->code; if (esr & ESR_ELx_FnV) info.si_addr = NULL; else info.si_addr = (void __user *)addr; - arm64_notify_die("", regs, &info, esr); + arm64_notify_die(inf->name, regs, &info, esr); return 0; } static const struct fault_info fault_info[] = { - { do_bad, SIGBUS, BUS_FIXME, "ttbr address size fault" }, - { do_bad, SIGBUS, BUS_FIXME, "level 1 address size fault" }, - { do_bad, SIGBUS, BUS_FIXME, "level 2 address size fault" }, - { do_bad, SIGBUS, BUS_FIXME, "level 3 address size fault" }, + { do_bad, SIGKILL, SI_KERNEL, "ttbr address size fault" }, + { do_bad, SIGKILL, SI_KERNEL, "level 1 address size fault" }, + { do_bad, SIGKILL, SI_KERNEL, "level 2 address size fault" }, + { do_bad, SIGKILL, SI_KERNEL, "level 3 address size fault" }, { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 0 translation fault" }, { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" }, { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" }, { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 8" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 8" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 12" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 12" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" }, { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" }, - { do_sea, SIGBUS, BUS_FIXME, "synchronous external abort" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 17" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 18" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 19" }, - { do_sea, SIGBUS, BUS_FIXME, "level 0 (translation table walk)" }, - { do_sea, SIGBUS, BUS_FIXME, "level 1 (translation table walk)" }, - { do_sea, SIGBUS, BUS_FIXME, "level 2 (translation table walk)" }, - { do_sea, SIGBUS, BUS_FIXME, "level 3 (translation table walk)" }, - { do_sea, SIGBUS, BUS_FIXME, "synchronous parity or ECC error" }, // Reserved when RAS is implemented - { do_bad, SIGBUS, BUS_FIXME, "unknown 25" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 26" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 27" }, - { do_sea, SIGBUS, BUS_FIXME, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented - { do_sea, SIGBUS, BUS_FIXME, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented - { do_sea, SIGBUS, BUS_FIXME, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented - { do_sea, SIGBUS, BUS_FIXME, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented - { do_bad, SIGBUS, BUS_FIXME, "unknown 32" }, + { do_sea, SIGBUS, BUS_OBJERR, "synchronous external abort" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 17" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 18" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 19" }, + { do_sea, SIGKILL, SI_KERNEL, "level 0 (translation table walk)" }, + { do_sea, SIGKILL, SI_KERNEL, "level 1 (translation table walk)" }, + { do_sea, SIGKILL, SI_KERNEL, "level 2 (translation table walk)" }, + { do_sea, SIGKILL, SI_KERNEL, "level 3 (translation table walk)" }, + { do_sea, SIGBUS, BUS_OBJERR, "synchronous parity or ECC error" }, // Reserved when RAS is implemented + { do_bad, SIGKILL, SI_KERNEL, "unknown 25" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 26" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 27" }, + { do_sea, SIGKILL, SI_KERNEL, "level 0 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented + { do_sea, SIGKILL, SI_KERNEL, "level 1 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented + { do_sea, SIGKILL, SI_KERNEL, "level 2 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented + { do_sea, SIGKILL, SI_KERNEL, "level 3 synchronous parity error (translation table walk)" }, // Reserved when RAS is implemented + { do_bad, SIGKILL, SI_KERNEL, "unknown 32" }, { do_alignment_fault, SIGBUS, BUS_ADRALN, "alignment fault" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 34" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 35" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 36" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 37" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 38" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 39" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 40" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 41" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 42" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 43" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 44" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 45" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 46" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 47" }, - { do_bad, SIGBUS, BUS_FIXME, "TLB conflict abort" }, - { do_bad, SIGBUS, BUS_FIXME, "Unsupported atomic hardware update fault" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 50" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 51" }, - { do_bad, SIGBUS, BUS_FIXME, "implementation fault (lockdown abort)" }, - { do_bad, SIGBUS, BUS_FIXME, "implementation fault (unsupported exclusive)" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 54" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 55" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 56" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 57" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 58" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 59" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 60" }, - { do_bad, SIGBUS, BUS_FIXME, "section domain fault" }, - { do_bad, SIGBUS, BUS_FIXME, "page domain fault" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 63" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 34" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 35" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 36" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 37" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 38" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 39" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 40" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 41" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 42" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 43" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 44" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 45" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 46" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 47" }, + { do_bad, SIGKILL, SI_KERNEL, "TLB conflict abort" }, + { do_bad, SIGKILL, SI_KERNEL, "Unsupported atomic hardware update fault" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 50" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 51" }, + { do_bad, SIGKILL, SI_KERNEL, "implementation fault (lockdown abort)" }, + { do_bad, SIGBUS, BUS_OBJERR, "implementation fault (unsupported exclusive)" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 54" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 55" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 56" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 57" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 58" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 59" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 60" }, + { do_bad, SIGKILL, SI_KERNEL, "section domain fault" }, + { do_bad, SIGKILL, SI_KERNEL, "page domain fault" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 63" }, }; int handle_guest_sea(phys_addr_t addr, unsigned int esr) @@ -698,19 +681,17 @@ asmlinkage void __exception do_mem_abort(unsigned long addr, unsigned int esr, if (!inf->fn(addr, esr, regs)) return; - pr_alert("Unhandled fault: %s at 0x%016lx\n", - inf->name, addr); - - mem_abort_decode(esr); - - if (!user_mode(regs)) + if (!user_mode(regs)) { + pr_alert("Unhandled fault at 0x%016lx\n", addr); + mem_abort_decode(esr); show_pte(addr); + } info.si_signo = inf->sig; info.si_errno = 0; info.si_code = inf->code; info.si_addr = (void __user *)addr; - arm64_notify_die("", regs, &info, esr); + arm64_notify_die(inf->name, regs, &info, esr); } asmlinkage void __exception do_el0_irq_bp_hardening(void) @@ -741,7 +722,6 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, struct pt_regs *regs) { struct siginfo info; - struct task_struct *tsk = current; if (user_mode(regs)) { if (instruction_pointer(regs) > TASK_SIZE) @@ -749,17 +729,11 @@ asmlinkage void __exception do_sp_pc_abort(unsigned long addr, local_irq_enable(); } - if (show_unhandled_signals && unhandled_signal(tsk, SIGBUS)) - pr_info_ratelimited("%s[%d]: %s exception: pc=%p sp=%p\n", - tsk->comm, task_pid_nr(tsk), - esr_get_class_string(esr), (void *)regs->pc, - (void *)regs->sp); - info.si_signo = SIGBUS; info.si_errno = 0; info.si_code = BUS_ADRALN; info.si_addr = (void __user *)addr; - arm64_notify_die("Oops - SP/PC alignment exception", regs, &info, esr); + arm64_notify_die("SP/PC alignment exception", regs, &info, esr); } int __init early_brk64(unsigned long addr, unsigned int esr, @@ -774,11 +748,11 @@ static struct fault_info __refdata debug_fault_info[] = { { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware breakpoint" }, { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware single-step" }, { do_bad, SIGTRAP, TRAP_HWBKPT, "hardware watchpoint" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 3" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 3" }, { do_bad, SIGTRAP, TRAP_BRKPT, "aarch32 BKPT" }, - { do_bad, SIGTRAP, TRAP_FIXME, "aarch32 vector catch" }, + { do_bad, SIGKILL, SI_KERNEL, "aarch32 vector catch" }, { early_brk64, SIGTRAP, TRAP_BRKPT, "aarch64 BRK" }, - { do_bad, SIGBUS, BUS_FIXME, "unknown 7" }, + { do_bad, SIGKILL, SI_KERNEL, "unknown 7" }, }; void __init hook_debug_fault_code(int nr, @@ -814,14 +788,11 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, if (!inf->fn(addr, esr, regs)) { rv = 1; } else { - pr_alert("Unhandled debug exception: %s (0x%08x) at 0x%016lx\n", - inf->name, esr, addr); - info.si_signo = inf->sig; info.si_errno = 0; info.si_code = inf->code; info.si_addr = (void __user *)addr; - arm64_notify_die("", regs, &info, 0); + arm64_notify_die(inf->name, regs, &info, esr); rv = 0; } @@ -833,7 +804,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr, NOKPROBE_SYMBOL(do_debug_exception); #ifdef CONFIG_ARM64_PAN -int cpu_enable_pan(void *__unused) +void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused) { /* * We modify PSTATE. This won't work from irq context as the PSTATE @@ -843,6 +814,5 @@ int cpu_enable_pan(void *__unused) config_sctlr_el1(SCTLR_EL1_SPAN, 0); asm(SET_PSTATE_PAN(1)); - return 0; } #endif /* CONFIG_ARM64_PAN */ diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index c0af47617299..5f9a73a4452c 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -36,6 +36,12 @@ #define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K #endif +#ifdef CONFIG_RANDOMIZE_BASE +#define TCR_KASLR_FLAGS TCR_NFD1 +#else +#define TCR_KASLR_FLAGS 0 +#endif + #define TCR_SMP_FLAGS TCR_SHARED /* PTWs cacheable, inner/outer WBWA */ @@ -432,7 +438,8 @@ ENTRY(__cpu_setup) * both user and kernel. */ ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ - TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 | TCR_A1 + TCR_TG_FLAGS | TCR_KASLR_FLAGS | TCR_ASID16 | \ + TCR_TBI0 | TCR_A1 tcr_set_idmap_t0sz x10, x9 /* @@ -441,16 +448,15 @@ ENTRY(__cpu_setup) tcr_compute_pa_size x10, #TCR_IPS_SHIFT, x5, x6 #ifdef CONFIG_ARM64_HW_AFDBM /* - * Hardware update of the Access and Dirty bits. + * Enable hardware update of the Access Flags bit. + * Hardware dirty bit management is enabled later, + * via capabilities. */ mrs x9, ID_AA64MMFR1_EL1 and x9, x9, #0xf - cbz x9, 2f - cmp x9, #2 - b.lt 1f - orr x10, x10, #TCR_HD // hardware Dirty flag update -1: orr x10, x10, #TCR_HA // hardware Access flag update -2: + cbz x9, 1f + orr x10, x10, #TCR_HA // hardware Access flag update +1: #endif /* CONFIG_ARM64_HW_AFDBM */ msr tcr_el1, x10 ret // return to head.S |