diff options
Diffstat (limited to 'src/arch')
251 files changed, 8158 insertions, 3673 deletions
diff --git a/src/arch/arm/Makefile b/src/arch/arm/Makefile index b6509dda0..f827ed422 100644 --- a/src/arch/arm/Makefile +++ b/src/arch/arm/Makefile @@ -3,9 +3,9 @@ ASM_TCHAR := % ASM_TCHAR_OPS := %% -# Include common ARM headers +# Include ARM-specific headers # -INCDIRS += arch/arm/include +INCDIRS := arch/$(ARCH)/include arch/arm/include $(INCDIRS) # ARM-specific directories containing source files # diff --git a/src/arch/arm/include/bits/acpi.h b/src/arch/arm/include/bits/acpi.h deleted file mode 100644 index f9f2f00e7..000000000 --- a/src/arch/arm/include/bits/acpi.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_ACPI_H -#define _BITS_ACPI_H - -/** @file - * - * ARM-specific ACPI API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_ACPI_H */ diff --git a/src/arch/arm/include/bits/hyperv.h b/src/arch/arm/include/bits/hyperv.h deleted file mode 100644 index f0e0c8793..000000000 --- a/src/arch/arm/include/bits/hyperv.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_HYPERV_H -#define _BITS_HYPERV_H - -/** @file - * - * Hyper-V interface - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_HYPERV_H */ diff --git a/src/arch/arm/include/bits/iomap.h b/src/arch/arm/include/bits/iomap.h deleted file mode 100644 index ae953c450..000000000 --- a/src/arch/arm/include/bits/iomap.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_IOMAP_H -#define _BITS_IOMAP_H - -/** @file - * - * ARM-specific I/O mapping API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_IOMAP_H */ diff --git a/src/arch/arm/include/bits/mp.h b/src/arch/arm/include/bits/mp.h deleted file mode 100644 index e7d4c0c16..000000000 --- a/src/arch/arm/include/bits/mp.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_MP_H -#define _BITS_MP_H - -/** @file - * - * ARM-specific multiprocessor API implementation - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_MP_H */ diff --git a/src/arch/arm/include/bits/nap.h b/src/arch/arm/include/bits/nap.h index e30a7146b..dbdf37166 100644 --- a/src/arch/arm/include/bits/nap.h +++ b/src/arch/arm/include/bits/nap.h @@ -9,6 +9,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); -#include <ipxe/efi/efiarm_nap.h> +/** + * Sleep until next CPU interrupt + * + */ +static inline __attribute__ (( always_inline )) void cpu_halt ( void ) { + __asm__ __volatile__ ( "wfi" ); +} -#endif /* _BITS_MAP_H */ +#endif /* _BITS_NAP_H */ diff --git a/src/arch/arm/include/bits/pci_io.h b/src/arch/arm/include/bits/pci_io.h deleted file mode 100644 index 91f507a44..000000000 --- a/src/arch/arm/include/bits/pci_io.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_PCI_IO_H -#define _BITS_PCI_IO_H - -/** @file - * - * ARM PCI I/O API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_PCI_IO_H */ diff --git a/src/arch/arm/include/bits/sanboot.h b/src/arch/arm/include/bits/sanboot.h deleted file mode 100644 index abd4c79a5..000000000 --- a/src/arch/arm/include/bits/sanboot.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_SANBOOT_H -#define _BITS_SANBOOT_H - -/** @file - * - * ARM-specific sanboot API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_SANBOOT_H */ diff --git a/src/arch/arm/include/bits/smbios.h b/src/arch/arm/include/bits/smbios.h deleted file mode 100644 index d94218116..000000000 --- a/src/arch/arm/include/bits/smbios.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_SMBIOS_H -#define _BITS_SMBIOS_H - -/** @file - * - * ARM-specific SMBIOS API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_SMBIOS_H */ diff --git a/src/arch/arm/include/bits/time.h b/src/arch/arm/include/bits/time.h deleted file mode 100644 index 724d8b932..000000000 --- a/src/arch/arm/include/bits/time.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_TIME_H -#define _BITS_TIME_H - -/** @file - * - * ARM-specific time API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_TIME_H */ diff --git a/src/arch/arm/include/bits/uaccess.h b/src/arch/arm/include/bits/uaccess.h deleted file mode 100644 index 87f11509c..000000000 --- a/src/arch/arm/include/bits/uaccess.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_UACCESS_H -#define _BITS_UACCESS_H - -/** @file - * - * ARM-specific user access API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_UACCESS_H */ diff --git a/src/arch/arm/include/bits/uart.h b/src/arch/arm/include/bits/uart.h deleted file mode 100644 index 6f85975f7..000000000 --- a/src/arch/arm/include/bits/uart.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_UART_H -#define _BITS_UART_H - -/** @file - * - * 16550-compatible UART - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_UART_H */ diff --git a/src/arch/arm/include/bits/umalloc.h b/src/arch/arm/include/bits/umalloc.h deleted file mode 100644 index 27970d7b2..000000000 --- a/src/arch/arm/include/bits/umalloc.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_UMALLOC_H -#define _BITS_UMALLOC_H - -/** @file - * - * ARM-specific user memory allocation API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_UMALLOC_H */ diff --git a/src/arch/arm/include/ipxe/efi/efiarm_nap.h b/src/arch/arm/include/ipxe/efi/efiarm_nap.h deleted file mode 100644 index dcbdd3e20..000000000 --- a/src/arch/arm/include/ipxe/efi/efiarm_nap.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _IPXE_EFIARM_NAP_H -#define _IPXE_EFIARM_NAP_H - -/** @file - * - * EFI CPU sleeping - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#ifdef NAP_EFIARM -#define NAP_PREFIX_efiarm -#else -#define NAP_PREFIX_efiarm __efiarm_ -#endif - -#endif /* _IPXE_EFIARM_NAP_H */ diff --git a/src/arch/arm32/Makefile b/src/arch/arm32/Makefile index 0c1cf99d1..070041734 100644 --- a/src/arch/arm32/Makefile +++ b/src/arch/arm32/Makefile @@ -1,3 +1,7 @@ +# Specify compressor +# +ZBIN = $(ZBIN32) + # ARM32-specific directories containing source files # SRCDIRS += arch/arm32/core diff --git a/src/arch/arm32/Makefile.efi b/src/arch/arm32/Makefile.efi index d720f34f0..9bd34383d 100644 --- a/src/arch/arm32/Makefile.efi +++ b/src/arch/arm32/Makefile.efi @@ -8,10 +8,6 @@ CFLAGS += -mfloat-abi=soft # ELF2EFI = $(ELF2EFI32) -# Specify EFI boot file -# -EFI_BOOT_FILE = bootarm.efi - # Include generic EFI Makefile # MAKEDEPS += arch/arm/Makefile.efi diff --git a/src/arch/arm32/Makefile.linux b/src/arch/arm32/Makefile.linux new file mode 100644 index 000000000..289118f29 --- /dev/null +++ b/src/arch/arm32/Makefile.linux @@ -0,0 +1,25 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# The number of different ABIs for 32-bit ARM is insane. It is +# unclear whether or not unaligned accesses ought to work in a 32-bit +# Linux userspace binary. When running in QEMU, unaligned accesses +# result in a SIGBUS. Since this is likely to be the most common use +# case (for running self-tests on an x86 build machine), and since we +# don't particularly care about performance for Linux userspace +# binaries, force the compiler to never generate an unaligned access. +# +CFLAGS += -mno-unaligned-access + +# Inhibit the harmless warning about wchar_t size mismatch between the +# linux_api.o helper object and the rest of iPXE. +# +LINUX_CFLAGS += -Wl,--no-wchar-size-warning + +# Starting virtual address +# +LDFLAGS += -Ttext=0x10000 + +# Include generic Linux Makefile +# +MAKEDEPS += arch/arm/Makefile.linux +include arch/arm/Makefile.linux diff --git a/src/arch/arm32/core/arm32_bigint.c b/src/arch/arm32/core/arm32_bigint.c deleted file mode 100644 index 29fb40a7c..000000000 --- a/src/arch/arm32/core/arm32_bigint.c +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - * - * You can also choose to distribute this program under the terms of - * the Unmodified Binary Distribution Licence (as given in the file - * COPYING.UBDL), provided that you have satisfied its requirements. - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <stdint.h> -#include <string.h> -#include <ipxe/bigint.h> - -/** @file - * - * Big integer support - */ - -/** - * Multiply big integers - * - * @v multiplicand0 Element 0 of big integer to be multiplied - * @v multiplicand_size Number of elements in multiplicand - * @v multiplier0 Element 0 of big integer to be multiplied - * @v multiplier_size Number of elements in multiplier - * @v result0 Element 0 of big integer to hold result - */ -void bigint_multiply_raw ( const uint32_t *multiplicand0, - unsigned int multiplicand_size, - const uint32_t *multiplier0, - unsigned int multiplier_size, - uint32_t *result0 ) { - unsigned int result_size = ( multiplicand_size + multiplier_size ); - const bigint_t ( multiplicand_size ) __attribute__ (( may_alias )) - *multiplicand = ( ( const void * ) multiplicand0 ); - const bigint_t ( multiplier_size ) __attribute__ (( may_alias )) - *multiplier = ( ( const void * ) multiplier0 ); - bigint_t ( result_size ) __attribute__ (( may_alias )) - *result = ( ( void * ) result0 ); - unsigned int i; - unsigned int j; - uint32_t multiplicand_element; - uint32_t multiplier_element; - uint32_t *result_elements; - uint32_t discard_low; - uint32_t discard_high; - uint32_t discard_temp; - - /* Zero result */ - memset ( result, 0, sizeof ( *result ) ); - - /* Multiply integers one element at a time */ - for ( i = 0 ; i < multiplicand_size ; i++ ) { - multiplicand_element = multiplicand->element[i]; - for ( j = 0 ; j < multiplier_size ; j++ ) { - multiplier_element = multiplier->element[j]; - result_elements = &result->element[ i + j ]; - /* Perform a single multiply, and add the - * resulting double-element into the result, - * carrying as necessary. The carry can - * never overflow beyond the end of the - * result, since: - * - * a < 2^{n}, b < 2^{m} => ab < 2^{n+m} - */ - __asm__ __volatile__ ( "umull %1, %2, %5, %6\n\t" - "ldr %3, [%0]\n\t" - "adds %3, %1\n\t" - "stmia %0!, {%3}\n\t" - "ldr %3, [%0]\n\t" - "adcs %3, %2\n\t" - "stmia %0!, {%3}\n\t" - "bcc 2f\n\t" - "\n1:\n\t" - "ldr %3, [%0]\n\t" - "adcs %3, #0\n\t" - "stmia %0!, {%3}\n\t" - "bcs 1b\n\t" - "\n2:\n\t" - : "+l" ( result_elements ), - "=l" ( discard_low ), - "=l" ( discard_high ), - "=l" ( discard_temp ), - "+m" ( *result ) - : "l" ( multiplicand_element ), - "l" ( multiplier_element ) - : "cc" ); - } - } -} diff --git a/src/arch/arm32/core/pmccntr.S b/src/arch/arm32/core/pmccntr.S new file mode 100644 index 000000000..c2fbeff4f --- /dev/null +++ b/src/arch/arm32/core/pmccntr.S @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * Performance Monitor Cycle Counter (PMCCNTR) + * + */ + + .section ".note.GNU-stack", "", %progbits + .text + .arm + +/* + * PMCCNTR status + * + * bit 31 set if PMCCNTR availability is not yet determined + * bit 0 set if PMCCNTR is available + * + */ + .section ".data.pmccntr_status", "aw", %progbits + .globl pmccntr_status +pmccntr_status: + .word 0x80000000 + +/* + * Check PMCCNTR availability + * + * Must preserve all registers, and return with either PMCCNTR enabled + * or the Z flag set to indicate unavailability. + * + */ + .section ".text.pmccntr_check", "ax", %progbits + .globl pmccntr_check + .type pmccntr_check, %function +pmccntr_check: + /* Save registers */ + stmfd sp!, { r0, r1 } + /* Read CPSR.M (bits 3:0, always permitted in PL0) */ + mrs r0, cpsr + and r0, r0, #0x0000000f + /* Read PMUSERENR.EN (bit 0, always permitted in PL0) */ + mrc p15, 0, r1, c9, c14, 0 + and r1, r1, #0x00000001 + /* Check if we are in PL1+ or in PL0 with PMUSERENR.EN set */ + orrs r0, r0, r1 + /* If PMCCNTR is unavailable, exit with status=0 and ZF set */ + beq 1f + /* Set PMCR.E (bit 0), set exit status=1 and ZF clear */ + movs r0, #0x00000001 + mcr p15, 0, r0, c9, c12, 0 + /* Set PMCNTENSET.C (bit 31) */ + mov r1, #0x80000000 + mcr p15, 0, r1, c9, c12, 1 +1: /* Store PMCCNTR status */ + ldr r1, pmccntr_status_ptr + str r0, [r1] + /* Restore registers and return */ + ldmfd sp!, { r0, r1 } + bx lr +pmccntr_status_ptr: + .word pmccntr_status + .size pmccntr_check, . - pmccntr_check diff --git a/src/arch/arm32/include/bits/bigint.h b/src/arch/arm32/include/bits/bigint.h index e4b511da7..988bef5ff 100644 --- a/src/arch/arm32/include/bits/bigint.h +++ b/src/arch/arm32/include/bits/bigint.h @@ -43,8 +43,9 @@ bigint_init_raw ( uint32_t *value0, unsigned int size, * @v addend0 Element 0 of big integer to add * @v value0 Element 0 of big integer to be added to * @v size Number of elements + * @ret carry Carry out */ -static inline __attribute__ (( always_inline )) void +static inline __attribute__ (( always_inline )) int bigint_add_raw ( const uint32_t *addend0, uint32_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = @@ -54,8 +55,9 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0, uint32_t *discard_end; uint32_t discard_addend_i; uint32_t discard_value_i; + int carry; - __asm__ __volatile__ ( "adds %2, %0, %8, lsl #2\n\t" /* clear CF */ + __asm__ __volatile__ ( "adds %2, %0, %9, lsl #2\n\t" /* clear CF */ "\n1:\n\t" "ldmia %0!, {%3}\n\t" "ldr %4, [%1]\n\t" @@ -68,9 +70,11 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0, "=l" ( discard_end ), "=l" ( discard_addend_i ), "=l" ( discard_value_i ), + "=@cccs" ( carry ), "+m" ( *value ) - : "0" ( addend0 ), "1" ( value0 ), "l" ( size ) - : "cc" ); + : "0" ( addend0 ), "1" ( value0 ), + "l" ( size ) ); + return carry; } /** @@ -79,8 +83,9 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0, * @v subtrahend0 Element 0 of big integer to subtract * @v value0 Element 0 of big integer to be subtracted from * @v size Number of elements + * @ret borrow Borrow out */ -static inline __attribute__ (( always_inline )) void +static inline __attribute__ (( always_inline )) int bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = @@ -90,8 +95,9 @@ bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0, uint32_t *discard_end; uint32_t discard_subtrahend_i; uint32_t discard_value_i; + int borrow; - __asm__ __volatile__ ( "add %2, %0, %8, lsl #2\n\t" + __asm__ __volatile__ ( "add %2, %0, %9, lsl #2\n\t" "cmp %2, %0\n\t" /* set CF */ "\n1:\n\t" "ldmia %0!, {%3}\n\t" @@ -105,27 +111,30 @@ bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0, "=l" ( discard_end ), "=l" ( discard_subtrahend_i ), "=l" ( discard_value_i ), + "=@cccc" ( borrow ), "+m" ( *value ) : "0" ( subtrahend0 ), "1" ( value0 ), - "l" ( size ) - : "cc" ); + "l" ( size ) ); + return borrow; } /** - * Rotate big integer left + * Shift big integer left * * @v value0 Element 0 of big integer * @v size Number of elements + * @ret out Bit shifted out */ -static inline __attribute__ (( always_inline )) void -bigint_rol_raw ( uint32_t *value0, unsigned int size ) { +static inline __attribute__ (( always_inline )) int +bigint_shl_raw ( uint32_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = ( ( void * ) value0 ); uint32_t *discard_value; uint32_t *discard_end; uint32_t discard_value_i; + int carry; - __asm__ __volatile__ ( "adds %1, %0, %5, lsl #2\n\t" /* clear CF */ + __asm__ __volatile__ ( "adds %1, %0, %1, lsl #2\n\t" /* clear CF */ "\n1:\n\t" "ldr %2, [%0]\n\t" "adcs %2, %2\n\t" @@ -135,26 +144,29 @@ bigint_rol_raw ( uint32_t *value0, unsigned int size ) { : "=l" ( discard_value ), "=l" ( discard_end ), "=l" ( discard_value_i ), + "=@cccs" ( carry ), "+m" ( *value ) - : "0" ( value0 ), "1" ( size ) - : "cc" ); + : "0" ( value0 ), "1" ( size ) ); + return carry; } /** - * Rotate big integer right + * Shift big integer right * * @v value0 Element 0 of big integer * @v size Number of elements + * @ret out Bit shifted out */ -static inline __attribute__ (( always_inline )) void -bigint_ror_raw ( uint32_t *value0, unsigned int size ) { +static inline __attribute__ (( always_inline )) int +bigint_shr_raw ( uint32_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = ( ( void * ) value0 ); uint32_t *discard_value; uint32_t *discard_end; uint32_t discard_value_i; + int carry; - __asm__ __volatile__ ( "adds %1, %0, %5, lsl #2\n\t" /* clear CF */ + __asm__ __volatile__ ( "adds %1, %0, %1, lsl #2\n\t" /* clear CF */ "\n1:\n\t" "ldmdb %1!, {%2}\n\t" "rrxs %2, %2\n\t" @@ -164,9 +176,10 @@ bigint_ror_raw ( uint32_t *value0, unsigned int size ) { : "=l" ( discard_value ), "=l" ( discard_end ), "=l" ( discard_value_i ), + "=@cccs" ( carry ), "+m" ( *value ) - : "0" ( value0 ), "1" ( size ) - : "cc" ); + : "0" ( value0 ), "1" ( size ) ); + return carry; } /** @@ -217,25 +230,6 @@ bigint_is_geq_raw ( const uint32_t *value0, const uint32_t *reference0, } /** - * Test if bit is set in big integer - * - * @v value0 Element 0 of big integer - * @v size Number of elements - * @v bit Bit to test - * @ret is_set Bit is set - */ -static inline __attribute__ (( always_inline )) int -bigint_bit_is_set_raw ( const uint32_t *value0, unsigned int size, - unsigned int bit ) { - const bigint_t ( size ) __attribute__ (( may_alias )) *value = - ( ( const void * ) value0 ); - unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) ); - unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) ); - - return ( value->element[index] & ( 1 << subindex ) ); -} - -/** * Find highest bit set in big integer * * @v value0 Element 0 of big integer @@ -309,10 +303,35 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused, *(--out_byte) = *(value_byte++); } -extern void bigint_multiply_raw ( const uint32_t *multiplicand0, - unsigned int multiplicand_size, - const uint32_t *multiplier0, - unsigned int multiplier_size, - uint32_t *value0 ); +/** + * Multiply big integer elements + * + * @v multiplicand Multiplicand element + * @v multiplier Multiplier element + * @v result Result element + * @v carry Carry element + */ +static inline __attribute__ (( always_inline )) void +bigint_multiply_one ( const uint32_t multiplicand, const uint32_t multiplier, + uint32_t *result, uint32_t *carry ) { + uint32_t discard_low; + uint32_t discard_high; + + __asm__ __volatile__ ( /* Perform multiplication */ + "umull %0, %1, %4, %5\n\t" + /* Accumulate result */ + "adds %2, %0\n\t" + "adc %1, #0\n\t" + /* Accumulate carry (cannot overflow) */ + "adds %2, %3\n\t" + "adc %3, %1, #0\n\t" + : "=r" ( discard_low ), + "=r" ( discard_high ), + "+r" ( *result ), + "+r" ( *carry ) + : "r" ( multiplicand ), + "r" ( multiplier ) + : "cc" ); +} #endif /* _BITS_BIGINT_H */ diff --git a/src/arch/arm32/include/bits/profile.h b/src/arch/arm32/include/bits/profile.h index 2b15d1604..31c321884 100644 --- a/src/arch/arm32/include/bits/profile.h +++ b/src/arch/arm32/include/bits/profile.h @@ -11,19 +11,30 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdint.h> +extern uint32_t pmccntr_status; + /** * Get profiling timestamp * * @ret timestamp Timestamp */ -static inline __attribute__ (( always_inline )) uint64_t +static inline __attribute__ (( always_inline )) unsigned long profile_timestamp ( void ) { uint32_t cycles; /* Read cycle counter */ - __asm__ __volatile__ ( "mcr p15, 0, %1, c9, c12, 0\n\t" - "mrc p15, 0, %0, c9, c13, 0\n\t" - : "=r" ( cycles ) : "r" ( 1 ) ); + __asm__ __volatile__ ( /* Check PMCCNTR status */ + "tst %0, %0\n\t" + /* Check availability if not yet known */ + "it mi\n\t" + "blxmi pmccntr_check\n\t" + /* Read PMCCNTR if available */ + "it ne\n\t" + "mrcne p15, 0, %0, c9, c13, 0\n\t" + "\n1:\n\t" + : "=r" ( cycles ) + : "0" ( pmccntr_status ) + : "cc", "lr" ); return cycles; } diff --git a/src/arch/arm32/include/setjmp.h b/src/arch/arm32/include/bits/setjmp.h index 4828b47a2..9ee264ecd 100644 --- a/src/arch/arm32/include/setjmp.h +++ b/src/arch/arm32/include/bits/setjmp.h @@ -1,5 +1,5 @@ -#ifndef _SETJMP_H -#define _SETJMP_H +#ifndef _BITS_SETJMP_H +#define _BITS_SETJMP_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); @@ -29,10 +29,4 @@ typedef struct { uint32_t lr; } jmp_buf[1]; -extern int __asmcall __attribute__ (( returns_twice )) -setjmp ( jmp_buf env ); - -extern void __asmcall __attribute__ (( noreturn )) -longjmp ( jmp_buf env, int val ); - -#endif /* _SETJMP_H */ +#endif /* _BITS_SETJMP_H */ diff --git a/src/arch/arm32/include/bits/tcpip.h b/src/arch/arm32/include/bits/tcpip.h deleted file mode 100644 index fc3c5b3ff..000000000 --- a/src/arch/arm32/include/bits/tcpip.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef _BITS_TCPIP_H -#define _BITS_TCPIP_H - -/** @file - * - * Transport-network layer interface - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -static inline __attribute__ (( always_inline )) uint16_t -tcpip_continue_chksum ( uint16_t partial, const void *data, size_t len ) { - - /* Not yet optimised */ - return generic_tcpip_continue_chksum ( partial, data, len ); -} - -#endif /* _BITS_TCPIP_H */ diff --git a/src/arch/arm32/include/gdbmach.h b/src/arch/arm32/include/gdbmach.h deleted file mode 100644 index cd152eedd..000000000 --- a/src/arch/arm32/include/gdbmach.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef GDBMACH_H -#define GDBMACH_H - -/** @file - * - * GDB architecture specifics - * - * This file declares functions for manipulating the machine state and - * debugging context. - * - */ - -#include <stdint.h> - -typedef unsigned long gdbreg_t; - -/* Register snapshot */ -enum { - /* Not yet implemented */ - GDBMACH_NREGS, -}; - -#define GDBMACH_SIZEOF_REGS ( GDBMACH_NREGS * sizeof ( gdbreg_t ) ) - -static inline void gdbmach_set_pc ( gdbreg_t *regs, gdbreg_t pc ) { - /* Not yet implemented */ - ( void ) regs; - ( void ) pc; -} - -static inline void gdbmach_set_single_step ( gdbreg_t *regs, int step ) { - /* Not yet implemented */ - ( void ) regs; - ( void ) step; -} - -static inline void gdbmach_breakpoint ( void ) { - /* Not yet implemented */ -} - -extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len, - int enable ); -extern void gdbmach_init ( void ); - -#endif /* GDBMACH_H */ diff --git a/src/arch/arm64/Makefile b/src/arch/arm64/Makefile index 9b9dd5ec8..719a1e61c 100644 --- a/src/arch/arm64/Makefile +++ b/src/arch/arm64/Makefile @@ -1,3 +1,7 @@ +# Specify compressor +# +ZBIN = $(ZBIN64) + # ARM64-specific directories containing source files # SRCDIRS += arch/arm64/core diff --git a/src/arch/arm64/Makefile.efi b/src/arch/arm64/Makefile.efi index 998a64d03..96f2953f4 100644 --- a/src/arch/arm64/Makefile.efi +++ b/src/arch/arm64/Makefile.efi @@ -4,10 +4,6 @@ # ELF2EFI = $(ELF2EFI64) -# Specify EFI boot file -# -EFI_BOOT_FILE = bootaa64.efi - # Include generic EFI Makefile # MAKEDEPS += arch/arm/Makefile.efi diff --git a/src/arch/arm64/core/arm64_bigint.c b/src/arch/arm64/core/arm64_bigint.c deleted file mode 100644 index 7740f1aef..000000000 --- a/src/arch/arm64/core/arm64_bigint.c +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - * - * You can also choose to distribute this program under the terms of - * the Unmodified Binary Distribution Licence (as given in the file - * COPYING.UBDL), provided that you have satisfied its requirements. - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <stdint.h> -#include <string.h> -#include <ipxe/bigint.h> - -/** @file - * - * Big integer support - */ - -/** - * Multiply big integers - * - * @v multiplicand0 Element 0 of big integer to be multiplied - * @v multiplicand_size Number of elements in multiplicand - * @v multiplier0 Element 0 of big integer to be multiplied - * @v multiplier_size Number of elements in multiplier - * @v result0 Element 0 of big integer to hold result - */ -void bigint_multiply_raw ( const uint64_t *multiplicand0, - unsigned int multiplicand_size, - const uint64_t *multiplier0, - unsigned int multiplier_size, - uint64_t *result0 ) { - unsigned int result_size = ( multiplicand_size + multiplier_size ); - const bigint_t ( multiplicand_size ) __attribute__ (( may_alias )) - *multiplicand = ( ( const void * ) multiplicand0 ); - const bigint_t ( multiplier_size ) __attribute__ (( may_alias )) - *multiplier = ( ( const void * ) multiplier0 ); - bigint_t ( result_size ) __attribute__ (( may_alias )) - *result = ( ( void * ) result0 ); - unsigned int i; - unsigned int j; - uint64_t multiplicand_element; - uint64_t multiplier_element; - uint64_t *result_elements; - uint64_t discard_low; - uint64_t discard_high; - uint64_t discard_temp_low; - uint64_t discard_temp_high; - - /* Zero result */ - memset ( result, 0, sizeof ( *result ) ); - - /* Multiply integers one element at a time */ - for ( i = 0 ; i < multiplicand_size ; i++ ) { - multiplicand_element = multiplicand->element[i]; - for ( j = 0 ; j < multiplier_size ; j++ ) { - multiplier_element = multiplier->element[j]; - result_elements = &result->element[ i + j ]; - /* Perform a single multiply, and add the - * resulting double-element into the result, - * carrying as necessary. The carry can - * never overflow beyond the end of the - * result, since: - * - * a < 2^{n}, b < 2^{m} => ab < 2^{n+m} - */ - __asm__ __volatile__ ( "mul %1, %6, %7\n\t" - "umulh %2, %6, %7\n\t" - "ldp %3, %4, [%0]\n\t" - "adds %3, %3, %1\n\t" - "adcs %4, %4, %2\n\t" - "stp %3, %4, [%0], #16\n\t" - "bcc 2f\n\t" - "\n1:\n\t" - "ldr %3, [%0]\n\t" - "adcs %3, %3, xzr\n\t" - "str %3, [%0], #8\n\t" - "bcs 1b\n\t" - "\n2:\n\t" - : "+r" ( result_elements ), - "=&r" ( discard_low ), - "=&r" ( discard_high ), - "=r" ( discard_temp_low ), - "=r" ( discard_temp_high ), - "+m" ( *result ) - : "r" ( multiplicand_element ), - "r" ( multiplier_element ) - : "cc" ); - } - } -} diff --git a/src/arch/arm64/core/arm64_string.c b/src/arch/arm64/core/arm64_string.c index 28a2b73bc..07a7eefdf 100644 --- a/src/arch/arm64/core/arm64_string.c +++ b/src/arch/arm64/core/arm64_string.c @@ -31,6 +31,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <string.h> +/** Block size (for "ldp"/"stp") */ +#define ARM64_STRING_BLKSZ 16 + /** * Copy memory area * @@ -40,59 +43,70 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * @ret dest Destination address */ void arm64_memcpy ( void *dest, const void *src, size_t len ) { - void *discard_dest; - void *discard_end; - const void *discard_src; - size_t discard_offset; + size_t len_pre; + size_t len_mid; + size_t len_post; unsigned long discard_data; unsigned long discard_low; unsigned long discard_high; + unsigned long discard_len; - /* If length is too short for an "ldp"/"stp" instruction pair, - * then just copy individual bytes. + /* Calculate pre-aligned, aligned, and post-aligned lengths. + * (Align on the destination address, on the assumption that + * misaligned stores are likely to be more expensive than + * misaligned loads.) */ - if ( len < 16 ) { - __asm__ __volatile__ ( "cbz %0, 2f\n\t" - "\n1:\n\t" - "sub %0, %0, #1\n\t" - "ldrb %w1, [%3, %0]\n\t" - "strb %w1, [%2, %0]\n\t" - "cbnz %0, 1b\n\t" - "\n2:\n\t" - : "=&r" ( discard_offset ), - "=&r" ( discard_data ) - : "r" ( dest ), "r" ( src ), "0" ( len ) - : "memory" ); - return; - } + len_pre = ( ( ARM64_STRING_BLKSZ - ( ( intptr_t ) dest ) ) & + ( ARM64_STRING_BLKSZ - 1 ) ); + if ( len_pre > len ) + len_pre = len; + len -= len_pre; + len_mid = ( len & ~( ARM64_STRING_BLKSZ - 1 ) ); + len -= len_mid; + len_post = len; - /* Use "ldp"/"stp" to copy 16 bytes at a time: one initial - * potentially unaligned access, multiple destination-aligned - * accesses, one final potentially unaligned access. - */ - __asm__ __volatile__ ( "ldp %3, %4, [%1], #16\n\t" - "stp %3, %4, [%0], #16\n\t" - "and %3, %0, #15\n\t" - "sub %0, %0, %3\n\t" - "sub %1, %1, %3\n\t" - "bic %2, %5, #15\n\t" - "b 2f\n\t" + /* Copy pre-aligned section */ + __asm__ __volatile__ ( "cbz %2, 2f\n\t" + "\n1:\n\t" + "ldrb %w3, [%1], #1\n\t" + "strb %w3, [%0], #1\n\t" + "sub %2, %2, #1\n\t" + "cbnz %2, 1b\n\t" + "\n2:\n\t" + : "+r" ( dest ), "+r" ( src ), + "=&r" ( discard_len ), + "=&r" ( discard_data ) + : "2" ( len_pre ) + : "memory" ); + + /* Copy aligned section */ + __asm__ __volatile__ ( "cbz %2, 2f\n\t" "\n1:\n\t" "ldp %3, %4, [%1], #16\n\t" "stp %3, %4, [%0], #16\n\t" + "sub %2, %2, #16\n\t" + "cbnz %2, 1b\n\t" "\n2:\n\t" - "cmp %0, %2\n\t" - "bne 1b\n\t" - "ldp %3, %4, [%6, #-16]\n\t" - "stp %3, %4, [%5, #-16]\n\t" - : "=&r" ( discard_dest ), - "=&r" ( discard_src ), - "=&r" ( discard_end ), + : "+r" ( dest ), "+r" ( src ), + "=&r" ( discard_len ), "=&r" ( discard_low ), "=&r" ( discard_high ) - : "r" ( dest + len ), "r" ( src + len ), - "0" ( dest ), "1" ( src ) - : "memory", "cc" ); + : "2" ( len_mid ) + : "memory" ); + + /* Copy post-aligned section */ + __asm__ __volatile__ ( "cbz %2, 2f\n\t" + "\n1:\n\t" + "ldrb %w3, [%1], #1\n\t" + "strb %w3, [%0], #1\n\t" + "sub %2, %2, #1\n\t" + "cbnz %2, 1b\n\t" + "\n2:\n\t" + : "+r" ( dest ), "+r" ( src ), + "=&r" ( discard_len ), + "=&r" ( discard_data ) + : "2" ( len_post ) + : "memory" ); } /** @@ -102,44 +116,56 @@ void arm64_memcpy ( void *dest, const void *src, size_t len ) { * @v len Length */ void arm64_bzero ( void *dest, size_t len ) { - size_t discard_offset; - void *discard_dest; - void *discard_end; + size_t len_pre; + size_t len_mid; + size_t len_post; + unsigned long discard_len; - /* If length is too short for an "stp" instruction, then just - * zero individual bytes. - */ - if ( len < 16 ) { - __asm__ __volatile__ ( "cbz %0, 2f\n\t" - "\n1:\n\t" - "sub %0, %0, #1\n\t" - "strb wzr, [%1, %0]\n\t" - "cbnz %0, 1b\n\t" - "\n2:\n\t" - : "=&r" ( discard_offset ) - : "r" ( dest ), "0" ( len ) - : "memory" ); - return; - } + /* Calculate pre-aligned, aligned, and post-aligned lengths */ + len_pre = ( ( ARM64_STRING_BLKSZ - ( ( intptr_t ) dest ) ) & + ( ARM64_STRING_BLKSZ - 1 ) ); + if ( len_pre > len ) + len_pre = len; + len -= len_pre; + len_mid = ( len & ~( ARM64_STRING_BLKSZ - 1 ) ); + len -= len_mid; + len_post = len; - /* Use "stp" to zero 16 bytes at a time: one initial - * potentially unaligned access, multiple aligned accesses, - * one final potentially unaligned access. - */ - __asm__ __volatile__ ( "stp xzr, xzr, [%0], #16\n\t" - "bic %0, %0, #15\n\t" - "bic %1, %2, #15\n\t" - "b 2f\n\t" + /* Zero pre-aligned section */ + __asm__ __volatile__ ( "cbz %1, 2f\n\t" + "\n1:\n\t" + "strb wzr, [%0], #1\n\t" + "sub %1, %1, #1\n\t" + "cbnz %1, 1b\n\t" + "\n2:\n\t" + : "+r" ( dest ), + "=&r" ( discard_len ) + : "1" ( len_pre ) + : "memory" ); + + /* Zero aligned section */ + __asm__ __volatile__ ( "cbz %1, 2f\n\t" "\n1:\n\t" "stp xzr, xzr, [%0], #16\n\t" + "sub %1, %1, #16\n\t" + "cbnz %1, 1b\n\t" "\n2:\n\t" - "cmp %0, %1\n\t" - "bne 1b\n\t" - "stp xzr, xzr, [%2, #-16]\n\t" - : "=&r" ( discard_dest ), - "=&r" ( discard_end ) - : "r" ( dest + len ), "0" ( dest ) - : "memory", "cc" ); + : "+r" ( dest ), + "=&r" ( discard_len ) + : "1" ( len_mid ) + : "memory" ); + + /* Zero post-aligned section */ + __asm__ __volatile__ ( "cbz %1, 2f\n\t" + "\n1:\n\t" + "strb wzr, [%0], #1\n\t" + "sub %1, %1, #1\n\t" + "cbnz %1, 1b\n\t" + "\n2:\n\t" + : "+r" ( dest ), + "=&r" ( discard_len ) + : "1" ( len_post ) + : "memory" ); } /** diff --git a/src/arch/arm64/include/bits/bigint.h b/src/arch/arm64/include/bits/bigint.h index 0d08bbd65..f4032e335 100644 --- a/src/arch/arm64/include/bits/bigint.h +++ b/src/arch/arm64/include/bits/bigint.h @@ -43,8 +43,9 @@ bigint_init_raw ( uint64_t *value0, unsigned int size, * @v addend0 Element 0 of big integer to add * @v value0 Element 0 of big integer to be added to * @v size Number of elements + * @ret carry Carry out */ -static inline __attribute__ (( always_inline )) void +static inline __attribute__ (( always_inline )) int bigint_add_raw ( const uint64_t *addend0, uint64_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = @@ -54,6 +55,7 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0, uint64_t discard_addend_i; uint64_t discard_value_i; unsigned int discard_size; + int carry; __asm__ __volatile__ ( "cmn xzr, xzr\n\t" /* clear CF */ "\n1:\n\t" @@ -68,9 +70,11 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0, "=r" ( discard_size ), "=r" ( discard_addend_i ), "=r" ( discard_value_i ), + "=@cccs" ( carry ), "+m" ( *value ) - : "0" ( addend0 ), "1" ( value0 ), "2" ( size ) - : "cc" ); + : "0" ( addend0 ), "1" ( value0 ), + "2" ( size ) ); + return carry; } /** @@ -79,8 +83,9 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0, * @v subtrahend0 Element 0 of big integer to subtract * @v value0 Element 0 of big integer to be subtracted from * @v size Number of elements + * @ret borrow Borrow out */ -static inline __attribute__ (( always_inline )) void +static inline __attribute__ (( always_inline )) int bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = @@ -90,6 +95,7 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0, uint64_t discard_subtrahend_i; uint64_t discard_value_i; unsigned int discard_size; + int borrow; __asm__ __volatile__ ( "cmp xzr, xzr\n\t" /* set CF */ "\n1:\n\t" @@ -104,25 +110,28 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0, "=r" ( discard_size ), "=r" ( discard_subtrahend_i ), "=r" ( discard_value_i ), + "=@cccc" ( borrow ), "+m" ( *value ) : "0" ( subtrahend0 ), "1" ( value0 ), - "2" ( size ) - : "cc" ); + "2" ( size ) ); + return borrow; } /** - * Rotate big integer left + * Shift big integer left * * @v value0 Element 0 of big integer * @v size Number of elements + * @ret out Bit shifted out */ -static inline __attribute__ (( always_inline )) void -bigint_rol_raw ( uint64_t *value0, unsigned int size ) { +static inline __attribute__ (( always_inline )) int +bigint_shl_raw ( uint64_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = ( ( void * ) value0 ); uint64_t *discard_value; uint64_t discard_value_i; unsigned int discard_size; + int carry; __asm__ __volatile__ ( "cmn xzr, xzr\n\t" /* clear CF */ "\n1:\n\t" @@ -134,40 +143,43 @@ bigint_rol_raw ( uint64_t *value0, unsigned int size ) { : "=r" ( discard_value ), "=r" ( discard_size ), "=r" ( discard_value_i ), + "=@cccs" ( carry ), "+m" ( *value ) - : "0" ( value0 ), "1" ( size ) - : "cc" ); + : "0" ( value0 ), "1" ( size ) ); + return carry; } /** - * Rotate big integer right + * Shift big integer right * * @v value0 Element 0 of big integer * @v size Number of elements + * @ret out Bit shifted out */ -static inline __attribute__ (( always_inline )) void -bigint_ror_raw ( uint64_t *value0, unsigned int size ) { +static inline __attribute__ (( always_inline )) int +bigint_shr_raw ( uint64_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = ( ( void * ) value0 ); uint64_t *discard_value; - uint64_t discard_value_i; - uint64_t discard_value_j; + uint64_t discard_high; unsigned int discard_size; + uint64_t low; - __asm__ __volatile__ ( "mov %3, #0\n\t" + __asm__ __volatile__ ( "mov %2, #0\n\t" "\n1:\n\t" "sub %w1, %w1, #1\n\t" - "ldr %2, [%0, %1, lsl #3]\n\t" - "extr %3, %3, %2, #1\n\t" - "str %3, [%0, %1, lsl #3]\n\t" - "mov %3, %2\n\t" + "ldr %3, [%0, %1, lsl #3]\n\t" + "extr %2, %2, %3, #1\n\t" + "str %2, [%0, %1, lsl #3]\n\t" + "mov %2, %3\n\t" "cbnz %w1, 1b\n\t" : "=r" ( discard_value ), "=r" ( discard_size ), - "=r" ( discard_value_i ), - "=r" ( discard_value_j ), + "=r" ( discard_high ), + "=r" ( low ), "+m" ( *value ) : "0" ( value0 ), "1" ( size ) ); + return ( low & 1 ); } /** @@ -218,25 +230,6 @@ bigint_is_geq_raw ( const uint64_t *value0, const uint64_t *reference0, } /** - * Test if bit is set in big integer - * - * @v value0 Element 0 of big integer - * @v size Number of elements - * @v bit Bit to test - * @ret is_set Bit is set - */ -static inline __attribute__ (( always_inline )) int -bigint_bit_is_set_raw ( const uint64_t *value0, unsigned int size, - unsigned int bit ) { - const bigint_t ( size ) __attribute__ (( may_alias )) *value = - ( ( const void * ) value0 ); - unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) ); - unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) ); - - return ( !! ( value->element[index] & ( 1UL << subindex ) ) ); -} - -/** * Find highest bit set in big integer * * @v value0 Element 0 of big integer @@ -310,10 +303,36 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused, *(--out_byte) = *(value_byte++); } -extern void bigint_multiply_raw ( const uint64_t *multiplicand0, - unsigned int multiplicand_size, - const uint64_t *multiplier0, - unsigned int multiplier_size, - uint64_t *value0 ); +/** + * Multiply big integer elements + * + * @v multiplicand Multiplicand element + * @v multiplier Multiplier element + * @v result Result element + * @v carry Carry element + */ +static inline __attribute__ (( always_inline )) void +bigint_multiply_one ( const uint64_t multiplicand, const uint64_t multiplier, + uint64_t *result, uint64_t *carry ) { + uint64_t discard_low; + uint64_t discard_high; + + __asm__ __volatile__ ( /* Perform multiplication */ + "mul %0, %4, %5\n\t" + "umulh %1, %4, %5\n\t" + /* Accumulate result */ + "adds %2, %2, %0\n\t" + "adc %1, %1, xzr\n\t" + /* Accumulate carry (cannot overflow) */ + "adds %2, %2, %3\n\t" + "adc %3, %1, xzr\n\t" + : "=&r" ( discard_low ), + "=r" ( discard_high ), + "+r" ( *result ), + "+r" ( *carry ) + : "r" ( multiplicand ), + "r" ( multiplier ) + : "cc" ); +} #endif /* _BITS_BIGINT_H */ diff --git a/src/arch/arm64/include/bits/lkrn.h b/src/arch/arm64/include/bits/lkrn.h new file mode 100644 index 000000000..943464e9b --- /dev/null +++ b/src/arch/arm64/include/bits/lkrn.h @@ -0,0 +1,30 @@ +#ifndef _BITS_LKRN_H +#define _BITS_LKRN_H + +/** @file + * + * Linux kernel image invocation + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** Header magic value */ +#define LKRN_MAGIC_ARCH LKRN_MAGIC_AARCH64 + +/** + * Jump to kernel entry point + * + * @v entry Kernel entry point + * @v fdt Device tree + */ +static inline __attribute__ (( noreturn )) void +lkrn_jump ( physaddr_t entry, physaddr_t fdt ) { + register unsigned long x0 asm ( "x0" ) = fdt; + + __asm__ __volatile__ ( "br %1" + : : "r" ( x0 ), "r" ( entry ) ); + __builtin_unreachable(); +} + +#endif /* _BITS_LKRN_H */ diff --git a/src/arch/arm64/include/bits/profile.h b/src/arch/arm64/include/bits/profile.h index 62ffa3772..4a5b3f7a1 100644 --- a/src/arch/arm64/include/bits/profile.h +++ b/src/arch/arm64/include/bits/profile.h @@ -16,7 +16,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * * @ret timestamp Timestamp */ -static inline __attribute__ (( always_inline )) uint64_t +static inline __attribute__ (( always_inline )) unsigned long profile_timestamp ( void ) { uint64_t cycles; diff --git a/src/arch/arm64/include/setjmp.h b/src/arch/arm64/include/bits/setjmp.h index 85a7a9cad..6ffd2fb0a 100644 --- a/src/arch/arm64/include/setjmp.h +++ b/src/arch/arm64/include/bits/setjmp.h @@ -1,5 +1,5 @@ -#ifndef _SETJMP_H -#define _SETJMP_H +#ifndef _BITS_SETJMP_H +#define _BITS_SETJMP_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); @@ -35,10 +35,4 @@ typedef struct { uint64_t sp; } jmp_buf[1]; -extern int __asmcall __attribute__ (( returns_twice )) -setjmp ( jmp_buf env ); - -extern void __asmcall __attribute__ (( noreturn )) -longjmp ( jmp_buf env, int val ); - -#endif /* _SETJMP_H */ +#endif /* _BITS_SETJMP_H */ diff --git a/src/arch/arm64/include/gdbmach.h b/src/arch/arm64/include/gdbmach.h deleted file mode 100644 index cd152eedd..000000000 --- a/src/arch/arm64/include/gdbmach.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef GDBMACH_H -#define GDBMACH_H - -/** @file - * - * GDB architecture specifics - * - * This file declares functions for manipulating the machine state and - * debugging context. - * - */ - -#include <stdint.h> - -typedef unsigned long gdbreg_t; - -/* Register snapshot */ -enum { - /* Not yet implemented */ - GDBMACH_NREGS, -}; - -#define GDBMACH_SIZEOF_REGS ( GDBMACH_NREGS * sizeof ( gdbreg_t ) ) - -static inline void gdbmach_set_pc ( gdbreg_t *regs, gdbreg_t pc ) { - /* Not yet implemented */ - ( void ) regs; - ( void ) pc; -} - -static inline void gdbmach_set_single_step ( gdbreg_t *regs, int step ) { - /* Not yet implemented */ - ( void ) regs; - ( void ) step; -} - -static inline void gdbmach_breakpoint ( void ) { - /* Not yet implemented */ -} - -extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len, - int enable ); -extern void gdbmach_init ( void ); - -#endif /* GDBMACH_H */ diff --git a/src/arch/i386/Makefile b/src/arch/i386/Makefile index e59f05fc8..44ccfccc8 100644 --- a/src/arch/i386/Makefile +++ b/src/arch/i386/Makefile @@ -1,3 +1,7 @@ +# Specify compressor +# +ZBIN = $(ZBIN32) + # Force i386-only instructions # CFLAGS += -march=i386 diff --git a/src/arch/i386/Makefile.efi b/src/arch/i386/Makefile.efi index 37ede65ac..aa809eb5d 100644 --- a/src/arch/i386/Makefile.efi +++ b/src/arch/i386/Makefile.efi @@ -8,10 +8,6 @@ ELF2EFI = $(ELF2EFI32) # CFLAGS += -malign-double -# Specify EFI boot file -# -EFI_BOOT_FILE = bootia32.efi - # Include generic EFI Makefile # MAKEDEPS += arch/x86/Makefile.efi diff --git a/src/arch/i386/include/gdbmach.h b/src/arch/i386/include/bits/gdbmach.h index 52cce7833..52cce7833 100644 --- a/src/arch/i386/include/gdbmach.h +++ b/src/arch/i386/include/bits/gdbmach.h diff --git a/src/arch/i386/include/bits/profile.h b/src/arch/i386/include/bits/profile.h index e184d7b51..21c216a81 100644 --- a/src/arch/i386/include/bits/profile.h +++ b/src/arch/i386/include/bits/profile.h @@ -16,12 +16,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * * @ret timestamp Timestamp */ -static inline __attribute__ (( always_inline )) uint64_t +static inline __attribute__ (( always_inline )) unsigned long profile_timestamp ( void ) { - uint64_t tsc; + uint32_t tsc; /* Read timestamp counter */ - __asm__ __volatile__ ( "rdtsc" : "=A" ( tsc ) ); + __asm__ __volatile__ ( "rdtsc" : "=a" ( tsc ) : : "edx" ); return tsc; } diff --git a/src/arch/i386/include/setjmp.h b/src/arch/i386/include/bits/setjmp.h index 98566696a..6b2ec9613 100644 --- a/src/arch/i386/include/setjmp.h +++ b/src/arch/i386/include/bits/setjmp.h @@ -1,5 +1,5 @@ -#ifndef _SETJMP_H -#define _SETJMP_H +#ifndef _BITS_SETJMP_H +#define _BITS_SETJMP_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); @@ -21,10 +21,4 @@ typedef struct { uint32_t ebp; } jmp_buf[1]; -extern int __asmcall __attribute__ (( returns_twice )) -setjmp ( jmp_buf env ); - -extern void __asmcall __attribute__ (( noreturn )) -longjmp ( jmp_buf env, int val ); - -#endif /* _SETJMP_H */ +#endif /* _BITS_SETJMP_H */ diff --git a/src/arch/i386/scripts/i386-kir.lds b/src/arch/i386/scripts/i386-kir.lds index 13c36f2bf..4cb656000 100644 --- a/src/arch/i386/scripts/i386-kir.lds +++ b/src/arch/i386/scripts/i386-kir.lds @@ -88,6 +88,8 @@ SECTIONS { __rodata16 = .; *(.rodata16) *(.rodata16.*) + *(.srodata) + *(.srodata.*) *(.rodata) *(.rodata.*) } @@ -95,6 +97,8 @@ SECTIONS { __data16 = .; *(.data16) *(.data16.*) + *(.sdata) + *(.sdata.*) *(.data) *(.data.*) KEEP(*(SORT(.tbl.*))) /* Various tables. See include/tables.h */ @@ -107,6 +111,8 @@ SECTIONS { _bss16 = .; *(.bss16) *(.bss16.*) + *(.sbss) + *(.sbss.*) *(.bss) *(.bss.*) *(COMMON) diff --git a/src/arch/loong64/Makefile b/src/arch/loong64/Makefile index fd0bf137f..809af07eb 100644 --- a/src/arch/loong64/Makefile +++ b/src/arch/loong64/Makefile @@ -1,3 +1,7 @@ +# Specify compressor +# +ZBIN = $(ZBIN64) + # Assembler section type character # ASM_TCHAR := @ @@ -18,6 +22,9 @@ endif # EFI requires -fshort-wchar, and nothing else currently uses wchar_t CFLAGS += -fshort-wchar +# Include LoongArch64-specific headers +INCDIRS := arch/$(ARCH)/include $(INCDIRS) + # LoongArch64-specific directories containing source files SRCDIRS += arch/loong64/core SRCDIRS += arch/loong64/interface/efi diff --git a/src/arch/loong64/Makefile.efi b/src/arch/loong64/Makefile.efi index 1c51bcd67..611c910f2 100644 --- a/src/arch/loong64/Makefile.efi +++ b/src/arch/loong64/Makefile.efi @@ -4,10 +4,6 @@ # ELF2EFI = $(ELF2EFI64) -# Specify EFI boot file -# -EFI_BOOT_FILE = bootloongarch64.efi - # Include generic EFI Makefile # MAKEDEPS += Makefile.efi diff --git a/src/arch/loong64/core/loong64_bigint.c b/src/arch/loong64/core/loong64_bigint.c deleted file mode 100644 index b428e22c3..000000000 --- a/src/arch/loong64/core/loong64_bigint.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>. - * Copyright (c) 2023, Xiaotian Wu <wuxiaotian@loongson.cn> - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - * - * You can also choose to distribute this program under the terms of - * the Unmodified Binary Distribution Licence (as given in the file - * COPYING.UBDL), provided that you have satisfied its requirements. - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <stdint.h> -#include <string.h> -#include <ipxe/bigint.h> - -/** @file - * - * Big integer support - */ - -/** - * Multiply big integers - * - * @v multiplicand0 Element 0 of big integer to be multiplied - * @v multiplicand_size Number of elements in multiplicand - * @v multiplier0 Element 0 of big integer to be multiplied - * @v multiplier_size Number of elements in multiplier - * @v result0 Element 0 of big integer to hold result - */ -void bigint_multiply_raw ( const uint64_t *multiplicand0, - unsigned int multiplicand_size, - const uint64_t *multiplier0, - unsigned int multiplier_size, - uint64_t *result0 ) { - unsigned int result_size = ( multiplicand_size + multiplier_size ); - const bigint_t ( multiplicand_size ) __attribute__ (( may_alias )) - *multiplicand = ( ( const void * ) multiplicand0 ); - const bigint_t ( multiplier_size ) __attribute__ (( may_alias )) - *multiplier = ( ( const void * ) multiplier0 ); - bigint_t ( result_size ) __attribute__ (( may_alias )) - *result = ( ( void * ) result0 ); - unsigned int i; - unsigned int j; - uint64_t multiplicand_element; - uint64_t multiplier_element; - uint64_t *result_elements; - uint64_t discard_low; - uint64_t discard_high; - uint64_t discard_temp_low; - uint64_t discard_temp_high; - - /* Zero result */ - memset ( result, 0, sizeof ( *result ) ); - - /* Multiply integers one element at a time */ - for ( i = 0 ; i < multiplicand_size ; i++ ) { - multiplicand_element = multiplicand->element[i]; - for ( j = 0 ; j < multiplier_size ; j++ ) { - multiplier_element = multiplier->element[j]; - result_elements = &result->element[ i + j ]; - /* Perform a single multiply, and add the - * resulting double-element into the result, - * carrying as necessary. The carry can - * never overflow beyond the end of the - * result, since: - * - * a < 2^{n}, b < 2^{m} => ab < 2^{n+m} - */ - __asm__ __volatile__ ( "mul.d %1, %6, %7\n\t" - "mulh.du %2, %6, %7\n\t" - - "ld.d %3, %0, 0\n\t" - "ld.d %4, %0, 8\n\t" - - "add.d %3, %3, %1\n\t" - "sltu $t0, %3, %1\n\t" - - "add.d %4, %4, %2\n\t" - "sltu $t1, %4, %2\n\t" - - "add.d %4, %4, $t0\n\t" - "sltu $t0, %4, $t0\n\t" - "or $t0, $t0, $t1\n\t" - - "st.d %3, %0, 0\n\t" - "st.d %4, %0, 8\n\t" - - "addi.d %0, %0, 16\n\t" - "beqz $t0, 2f\n" - "1:\n\t" - "ld.d %3, %0, 0\n\t" - "add.d %3, %3, $t0\n\t" - "sltu $t0, %3, $t0\n\t" - "st.d %3, %0, 0\n\t" - "addi.d %0, %0, 8\n\t" - "bnez $t0, 1b\n" - "2:" - : "+r" ( result_elements ), - "=&r" ( discard_low ), - "=&r" ( discard_high ), - "=r" ( discard_temp_low ), - "=r" ( discard_temp_high ), - "+m" ( *result ) - : "r" ( multiplicand_element ), - "r" ( multiplier_element ) - : "t0", "t1" ); - } - } -} diff --git a/src/arch/loong64/include/bits/acpi.h b/src/arch/loong64/include/bits/acpi.h deleted file mode 100644 index 83dd1df9a..000000000 --- a/src/arch/loong64/include/bits/acpi.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_ACPI_H -#define _BITS_ACPI_H - -/** @file - * - * LoongArch64-specific ACPI API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_ACPI_H */ diff --git a/src/arch/loong64/include/bits/bigint.h b/src/arch/loong64/include/bits/bigint.h index bec748beb..6a879503a 100644 --- a/src/arch/loong64/include/bits/bigint.h +++ b/src/arch/loong64/include/bits/bigint.h @@ -43,8 +43,9 @@ bigint_init_raw ( uint64_t *value0, unsigned int size, * @v addend0 Element 0 of big integer to add * @v value0 Element 0 of big integer to be added to * @v size Number of elements + * @ret carry Carry out */ -static inline __attribute__ (( always_inline )) void +static inline __attribute__ (( always_inline )) int bigint_add_raw ( const uint64_t *addend0, uint64_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = @@ -53,20 +54,20 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0, uint64_t *discard_value; uint64_t discard_addend_i; uint64_t discard_value_i; - uint64_t discard_carry; uint64_t discard_temp; unsigned int discard_size; + uint64_t carry; __asm__ __volatile__ ( "\n1:\n\t" /* Load addend[i] and value[i] */ "ld.d %3, %0, 0\n\t" "ld.d %4, %1, 0\n\t" /* Add carry flag and addend */ - "add.d %4, %4, %5\n\t" - "sltu %6, %4, %5\n\t" + "add.d %4, %4, %6\n\t" + "sltu %5, %4, %6\n\t" "add.d %4, %4, %3\n\t" - "sltu %5, %4, %3\n\t" - "or %5, %5, %6\n\t" + "sltu %6, %4, %3\n\t" + "or %6, %5, %6\n\t" /* Store value[i] */ "st.d %4, %1, 0\n\t" /* Loop */ @@ -79,11 +80,12 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0, "=r" ( discard_size ), "=r" ( discard_addend_i ), "=r" ( discard_value_i ), - "=r" ( discard_carry ), "=r" ( discard_temp ), + "=r" ( carry ), "+m" ( *value ) : "0" ( addend0 ), "1" ( value0 ), - "2" ( size ), "5" ( 0 ) ); + "2" ( size ), "6" ( 0 ) ); + return carry; } /** @@ -92,8 +94,9 @@ bigint_add_raw ( const uint64_t *addend0, uint64_t *value0, * @v subtrahend0 Element 0 of big integer to subtract * @v value0 Element 0 of big integer to be subtracted from * @v size Number of elements + * @ret borrow Borrow out */ -static inline __attribute__ (( always_inline )) void +static inline __attribute__ (( always_inline )) int bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = @@ -102,20 +105,20 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0, uint64_t *discard_value; uint64_t discard_subtrahend_i; uint64_t discard_value_i; - uint64_t discard_carry; uint64_t discard_temp; unsigned int discard_size; + uint64_t borrow; __asm__ __volatile__ ( "\n1:\n\t" /* Load subtrahend[i] and value[i] */ "ld.d %3, %0, 0\n\t" "ld.d %4, %1, 0\n\t" /* Subtract carry flag and subtrahend */ - "sltu %6, %4, %5\n\t" - "sub.d %4, %4, %5\n\t" - "sltu %5, %4, %3\n\t" + "sltu %5, %4, %6\n\t" + "sub.d %4, %4, %6\n\t" + "sltu %6, %4, %3\n\t" "sub.d %4, %4, %3\n\t" - "or %5, %5, %6\n\t" + "or %6, %5, %6\n\t" /* Store value[i] */ "st.d %4, %1, 0\n\t" /* Loop */ @@ -128,38 +131,40 @@ bigint_subtract_raw ( const uint64_t *subtrahend0, uint64_t *value0, "=r" ( discard_size ), "=r" ( discard_subtrahend_i ), "=r" ( discard_value_i ), - "=r" ( discard_carry ), "=r" ( discard_temp ), + "=r" ( borrow ), "+m" ( *value ) : "0" ( subtrahend0 ), "1" ( value0 ), - "2" ( size ), "5" ( 0 ) ); + "2" ( size ), "6" ( 0 ) ); + return borrow; } /** - * Rotate big integer left + * Shift big integer left * * @v value0 Element 0 of big integer * @v size Number of elements + * @ret out Bit shifted out */ -static inline __attribute__ (( always_inline )) void -bigint_rol_raw ( uint64_t *value0, unsigned int size ) { +static inline __attribute__ (( always_inline )) int +bigint_shl_raw ( uint64_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = ( ( void * ) value0 ); uint64_t *discard_value; uint64_t discard_value_i; - uint64_t discard_carry; uint64_t discard_temp; unsigned int discard_size; + uint64_t carry; __asm__ __volatile__ ( "\n1:\n\t" /* Load value[i] */ "ld.d %2, %0, 0\n\t" /* Shift left */ "rotri.d %2, %2, 63\n\t" - "andi %4, %2, 1\n\t" - "xor %2, %2, %4\n\t" - "or %2, %2, %3\n\t" - "move %3, %4\n\t" + "andi %3, %2, 1\n\t" + "xor %2, %2, %3\n\t" + "or %2, %2, %4\n\t" + "move %4, %3\n\t" /* Store value[i] */ "st.d %2, %0, 0\n\t" /* Loop */ @@ -169,37 +174,39 @@ bigint_rol_raw ( uint64_t *value0, unsigned int size ) { : "=r" ( discard_value ), "=r" ( discard_size ), "=r" ( discard_value_i ), - "=r" ( discard_carry ), "=r" ( discard_temp ), + "=r" ( carry ), "+m" ( *value ) - : "0" ( value0 ), "1" ( size ), "3" ( 0 ) + : "0" ( value0 ), "1" ( size ), "4" ( 0 ) : "cc" ); + return ( carry & 1 ); } /** - * Rotate big integer right + * Shift big integer right * * @v value0 Element 0 of big integer * @v size Number of elements + * @ret out Bit shifted out */ -static inline __attribute__ (( always_inline )) void -bigint_ror_raw ( uint64_t *value0, unsigned int size ) { +static inline __attribute__ (( always_inline )) int +bigint_shr_raw ( uint64_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = ( ( void * ) value0 ); uint64_t *discard_value; uint64_t discard_value_i; - uint64_t discard_carry; uint64_t discard_temp; unsigned int discard_size; + uint64_t carry; __asm__ __volatile__ ( "\n1:\n\t" /* Load value[i] */ "ld.d %2, %0, -8\n\t" /* Shift right */ - "andi %4, %2, 1\n\t" - "xor %2, %2, %4\n\t" - "or %2, %2, %3\n\t" - "move %3, %4\n\t" + "andi %3, %2, 1\n\t" + "xor %2, %2, %3\n\t" + "or %2, %2, %4\n\t" + "move %4, %3\n\t" "rotri.d %2, %2, 1\n\t" /* Store value[i] */ "st.d %2, %0, -8\n\t" @@ -210,11 +217,12 @@ bigint_ror_raw ( uint64_t *value0, unsigned int size ) { : "=r" ( discard_value ), "=r" ( discard_size ), "=r" ( discard_value_i ), - "=r" ( discard_carry ), "=r" ( discard_temp ), + "=r" ( carry ), "+m" ( *value ) - : "0" ( value0 + size ), "1" ( size ), "3" ( 0 ) + : "0" ( value0 + size ), "1" ( size ), "4" ( 0 ) : "cc" ); + return ( carry & 1 ); } /** @@ -265,25 +273,6 @@ bigint_is_geq_raw ( const uint64_t *value0, const uint64_t *reference0, } /** - * Test if bit is set in big integer - * - * @v value0 Element 0 of big integer - * @v size Number of elements - * @v bit Bit to test - * @ret is_set Bit is set - */ -static inline __attribute__ (( always_inline )) int -bigint_bit_is_set_raw ( const uint64_t *value0, unsigned int size, - unsigned int bit ) { - const bigint_t ( size ) __attribute__ (( may_alias )) *value = - ( ( const void * ) value0 ); - unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) ); - unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) ); - - return ( !! ( value->element[index] & ( 1UL << subindex ) ) ); -} - -/** * Find highest bit set in big integer * * @v value0 Element 0 of big integer @@ -357,10 +346,39 @@ bigint_done_raw ( const uint64_t *value0, unsigned int size __unused, *(--out_byte) = *(value_byte++); } -extern void bigint_multiply_raw ( const uint64_t *multiplicand0, - unsigned int multiplicand_size, - const uint64_t *multiplier0, - unsigned int multiplier_size, - uint64_t *value0 ); +/** + * Multiply big integer elements + * + * @v multiplicand Multiplicand element + * @v multiplier Multiplier element + * @v result Result element + * @v carry Carry element + */ +static inline __attribute__ (( always_inline )) void +bigint_multiply_one ( const uint64_t multiplicand, const uint64_t multiplier, + uint64_t *result, uint64_t *carry ) { + uint64_t discard_low; + uint64_t discard_high; + uint64_t discard_carry; + + __asm__ __volatile__ ( /* Perform multiplication */ + "mul.d %0, %5, %6\n\t" + "mulh.du %1, %5, %6\n\t" + /* Accumulate low half */ + "add.d %3, %3, %0\n\t" + "sltu %2, %3, %0\n\t" + "add.d %1, %1, %2\n\t" + /* Accumulate carry (cannot overflow) */ + "add.d %3, %3, %4\n\t" + "sltu %2, %3, %4\n\t" + "add.d %4, %1, %2\n\t" + : "=&r" ( discard_low ), + "=r" ( discard_high ), + "=r" ( discard_carry ), + "+r" ( *result ), + "+r" ( *carry ) + : "r" ( multiplicand ), + "r" ( multiplier ) ); +} #endif /* _BITS_BIGINT_H */ diff --git a/src/arch/loong64/include/bits/hyperv.h b/src/arch/loong64/include/bits/hyperv.h deleted file mode 100644 index f0e0c8793..000000000 --- a/src/arch/loong64/include/bits/hyperv.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_HYPERV_H -#define _BITS_HYPERV_H - -/** @file - * - * Hyper-V interface - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_HYPERV_H */ diff --git a/src/arch/loong64/include/bits/mp.h b/src/arch/loong64/include/bits/mp.h deleted file mode 100644 index fef2fd59a..000000000 --- a/src/arch/loong64/include/bits/mp.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_MP_H -#define _BITS_MP_H - -/** @file - * - * LoongArch64-specific multiprocessor API implementation - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_MP_H */ diff --git a/src/arch/loong64/include/bits/nap.h b/src/arch/loong64/include/bits/nap.h index 2deba3558..d904db537 100644 --- a/src/arch/loong64/include/bits/nap.h +++ b/src/arch/loong64/include/bits/nap.h @@ -9,6 +9,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); -#include <ipxe/efi/efiloong64_nap.h> +/** + * Sleep until next CPU interrupt + * + */ +static inline __attribute__ (( always_inline )) void cpu_halt ( void ) { + __asm__ __volatile__ ( "idle 0" ); +} #endif /* _BITS_NAP_H */ diff --git a/src/arch/loong64/include/bits/pci_io.h b/src/arch/loong64/include/bits/pci_io.h deleted file mode 100644 index fdc5141cf..000000000 --- a/src/arch/loong64/include/bits/pci_io.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_PCI_IO_H -#define _BITS_PCI_IO_H - -/** @file - * - * LoongArch64-specific PCI I/O API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_PCI_IO_H */ diff --git a/src/arch/loong64/include/bits/profile.h b/src/arch/loong64/include/bits/profile.h index 9f597ce2d..02f8d4b7c 100644 --- a/src/arch/loong64/include/bits/profile.h +++ b/src/arch/loong64/include/bits/profile.h @@ -16,7 +16,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * * @ret timestamp Timestamp */ -static inline __attribute__ (( always_inline )) uint64_t +static inline __attribute__ (( always_inline )) unsigned long profile_timestamp ( void ) { uint64_t cycles; diff --git a/src/arch/loong64/include/bits/reboot.h b/src/arch/loong64/include/bits/reboot.h deleted file mode 100644 index 96a1eb1ce..000000000 --- a/src/arch/loong64/include/bits/reboot.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_REBOOT_H -#define _BITS_REBOOT_H - -/** @file - * - * LoongArch64-specific reboot API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_REBOOT_H */ diff --git a/src/arch/loong64/include/bits/sanboot.h b/src/arch/loong64/include/bits/sanboot.h deleted file mode 100644 index f9205e2ad..000000000 --- a/src/arch/loong64/include/bits/sanboot.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_SANBOOT_H -#define _BITS_SANBOOT_H - -/** @file - * - * LoongArch64-specific sanboot API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_SANBOOT_H */ diff --git a/src/arch/loong64/include/bits/setjmp.h b/src/arch/loong64/include/bits/setjmp.h new file mode 100644 index 000000000..c8d7cef0e --- /dev/null +++ b/src/arch/loong64/include/bits/setjmp.h @@ -0,0 +1,23 @@ +#ifndef _BITS_SETJMP_H +#define _BITS_SETJMP_H + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** A jump buffer */ +typedef struct { + uint64_t s0; + uint64_t s1; + uint64_t s2; + uint64_t s3; + uint64_t s4; + uint64_t s5; + uint64_t s6; + uint64_t s7; + uint64_t s8; + + uint64_t fp; + uint64_t sp; + uint64_t ra; +} jmp_buf[1]; + +#endif /* _BITS_SETJMP_H */ diff --git a/src/arch/loong64/include/bits/smbios.h b/src/arch/loong64/include/bits/smbios.h deleted file mode 100644 index 6c87db430..000000000 --- a/src/arch/loong64/include/bits/smbios.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_SMBIOS_H -#define _BITS_SMBIOS_H - -/** @file - * - * LoongArch64-specific SMBIOS API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_SMBIOS_H */ diff --git a/src/arch/loong64/include/bits/tcpip.h b/src/arch/loong64/include/bits/tcpip.h deleted file mode 100644 index fc3c5b3ff..000000000 --- a/src/arch/loong64/include/bits/tcpip.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef _BITS_TCPIP_H -#define _BITS_TCPIP_H - -/** @file - * - * Transport-network layer interface - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -static inline __attribute__ (( always_inline )) uint16_t -tcpip_continue_chksum ( uint16_t partial, const void *data, size_t len ) { - - /* Not yet optimised */ - return generic_tcpip_continue_chksum ( partial, data, len ); -} - -#endif /* _BITS_TCPIP_H */ diff --git a/src/arch/loong64/include/bits/time.h b/src/arch/loong64/include/bits/time.h deleted file mode 100644 index 4cd7485cf..000000000 --- a/src/arch/loong64/include/bits/time.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_TIME_H -#define _BITS_TIME_H - -/** @file - * - * LoongArch64-specific time API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_TIME_H */ diff --git a/src/arch/loong64/include/bits/uaccess.h b/src/arch/loong64/include/bits/uaccess.h deleted file mode 100644 index dddd9be04..000000000 --- a/src/arch/loong64/include/bits/uaccess.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_UACCESS_H -#define _BITS_UACCESS_H - -/** @file - * - * LoongArch64-specific user access API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_UACCESS_H */ diff --git a/src/arch/loong64/include/bits/uart.h b/src/arch/loong64/include/bits/uart.h deleted file mode 100644 index 6f85975f7..000000000 --- a/src/arch/loong64/include/bits/uart.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_UART_H -#define _BITS_UART_H - -/** @file - * - * 16550-compatible UART - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_UART_H */ diff --git a/src/arch/loong64/include/bits/umalloc.h b/src/arch/loong64/include/bits/umalloc.h deleted file mode 100644 index f6978b8bd..000000000 --- a/src/arch/loong64/include/bits/umalloc.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef _BITS_UMALLOC_H -#define _BITS_UMALLOC_H - -/** @file - * - * LoongArch64-specific user memory allocation API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#endif /* _BITS_UMALLOC_H */ diff --git a/src/arch/loong64/include/bits/xen.h b/src/arch/loong64/include/bits/xen.h deleted file mode 100644 index 2a3d7741c..000000000 --- a/src/arch/loong64/include/bits/xen.h +++ /dev/null @@ -1,13 +0,0 @@ -#ifndef _BITS_XEN_H -#define _BITS_XEN_H - -/** @file - * - * Xen interface - * - */ -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <ipxe/nonxen.h> - -#endif /* _BITS_XEN_H */ diff --git a/src/arch/loong64/include/gdbmach.h b/src/arch/loong64/include/gdbmach.h deleted file mode 100644 index cd152eedd..000000000 --- a/src/arch/loong64/include/gdbmach.h +++ /dev/null @@ -1,45 +0,0 @@ -#ifndef GDBMACH_H -#define GDBMACH_H - -/** @file - * - * GDB architecture specifics - * - * This file declares functions for manipulating the machine state and - * debugging context. - * - */ - -#include <stdint.h> - -typedef unsigned long gdbreg_t; - -/* Register snapshot */ -enum { - /* Not yet implemented */ - GDBMACH_NREGS, -}; - -#define GDBMACH_SIZEOF_REGS ( GDBMACH_NREGS * sizeof ( gdbreg_t ) ) - -static inline void gdbmach_set_pc ( gdbreg_t *regs, gdbreg_t pc ) { - /* Not yet implemented */ - ( void ) regs; - ( void ) pc; -} - -static inline void gdbmach_set_single_step ( gdbreg_t *regs, int step ) { - /* Not yet implemented */ - ( void ) regs; - ( void ) step; -} - -static inline void gdbmach_breakpoint ( void ) { - /* Not yet implemented */ -} - -extern int gdbmach_set_breakpoint ( int type, unsigned long addr, size_t len, - int enable ); -extern void gdbmach_init ( void ); - -#endif /* GDBMACH_H */ diff --git a/src/arch/loong64/include/ipxe/efi/efiloong64_nap.h b/src/arch/loong64/include/ipxe/efi/efiloong64_nap.h deleted file mode 100644 index 5c0d38636..000000000 --- a/src/arch/loong64/include/ipxe/efi/efiloong64_nap.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _IPXE_EFILOONG64_NAP_H -#define _IPXE_EFILOONG64_NAP_H - -/** @file - * - * EFI CPU sleeping - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#ifdef NAP_EFILOONG64 -#define NAP_PREFIX_efiloong64 -#else -#define NAP_PREFIX_efiloong64 __efiloong64_ -#endif - -#endif /* _IPXE_EFILOONG64_NAP_H */ diff --git a/src/arch/loong64/include/setjmp.h b/src/arch/loong64/include/setjmp.h deleted file mode 100644 index 1e5168338..000000000 --- a/src/arch/loong64/include/setjmp.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef _SETJMP_H -#define _SETJMP_H - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <stdint.h> - -/** jump buffer env*/ -typedef struct { - uint64_t s0; - uint64_t s1; - uint64_t s2; - uint64_t s3; - uint64_t s4; - uint64_t s5; - uint64_t s6; - uint64_t s7; - uint64_t s8; - - uint64_t fp; - uint64_t sp; - uint64_t ra; -} jmp_buf[1]; - -extern int __asmcall __attribute__ (( returns_twice )) -setjmp ( jmp_buf env ); - -extern void __asmcall __attribute__ (( noreturn )) -longjmp ( jmp_buf env, int val ); - -#endif /* _SETJMP_H */ diff --git a/src/arch/riscv/Makefile b/src/arch/riscv/Makefile new file mode 100644 index 000000000..324e1403e --- /dev/null +++ b/src/arch/riscv/Makefile @@ -0,0 +1,22 @@ +# Assembler section type character +# +ASM_TCHAR := @ +ASM_TCHAR_OPS := @ + +# Include RISCV-specific headers +# +INCDIRS := arch/$(ARCH)/include arch/riscv/include $(INCDIRS) + +# RISCV-specific directories containing source files +# +SRCDIRS += arch/riscv/core +SRCDIRS += arch/riscv/interface/sbi +SRCDIRS += arch/riscv/prefix + +# RISCV-specific flags +# +CFLAGS += -mno-strict-align -mno-plt + +# EFI requires -fshort-wchar, and nothing else currently uses wchar_t +# +CFLAGS += -fshort-wchar diff --git a/src/arch/riscv/Makefile.efi b/src/arch/riscv/Makefile.efi new file mode 100644 index 000000000..957e8b884 --- /dev/null +++ b/src/arch/riscv/Makefile.efi @@ -0,0 +1,10 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# RISCV-specific flags +# +CFLAGS += -mcmodel=medany + +# Include generic EFI Makefile +# +MAKEDEPS += Makefile.efi +include Makefile.efi diff --git a/src/arch/riscv/Makefile.linux b/src/arch/riscv/Makefile.linux new file mode 100644 index 000000000..42590441e --- /dev/null +++ b/src/arch/riscv/Makefile.linux @@ -0,0 +1,6 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# Include generic Linux Makefile +# +MAKEDEPS += Makefile.linux +include Makefile.linux diff --git a/src/arch/riscv/Makefile.sbi b/src/arch/riscv/Makefile.sbi new file mode 100644 index 000000000..5e5dc9a6b --- /dev/null +++ b/src/arch/riscv/Makefile.sbi @@ -0,0 +1,31 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# Build a position-independent executable, with relocations required +# only for data values. Runtime relocations are applied by the +# prefix code. +# +CFLAGS += -mcmodel=medany -fpie +LDFLAGS += -pie --no-dynamic-linker -z combreloc + +# Place explicitly zero-initialised variables in the .data section +# rather than in .bss, so that we can rely on their values even during +# parsing of the system memory map prior to relocation (and therefore +# prior to explicit zeroing of the .bss section). +# +CFLAGS += -fno-zero-initialized-in-bss + +# Linker script +# +LDSCRIPT = arch/riscv/scripts/sbi.lds + +# Media types +# +MEDIA += sbi +MEDIA += lkrn + +# Padded flash device images (e.g. for QEMU's -pflash option) +# +NON_AUTO_MEDIA += pf32 +%.pf32 : %.sbi $(MAKEDEPS) + $(Q)$(CP) $< $@ + $(Q)$(TRUNCATE) -s 32M $@ diff --git a/src/arch/riscv/core/hart.c b/src/arch/riscv/core/hart.c new file mode 100644 index 000000000..d9cfb4e5d --- /dev/null +++ b/src/arch/riscv/core/hart.c @@ -0,0 +1,100 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** @file + * + * Hardware threads (harts) + * + */ + +#include <string.h> +#include <stdio.h> +#include <errno.h> +#include <ipxe/fdt.h> +#include <ipxe/hart.h> + +/** Boot hart ID */ +unsigned long boot_hart; + +/** Colour for debug messages */ +#define colour &boot_hart + +/** + * Find boot hart node + * + * @v offset Boot hart node offset + * @ret rc Return status code + */ +static int hart_node ( unsigned int *offset ) { + char path[27 /* "/cpus/cpu@XXXXXXXXXXXXXXXX" + NUL */ ]; + int rc; + + /* Construct node path */ + snprintf ( path, sizeof ( path ), "/cpus/cpu@%lx", boot_hart ); + + /* Find node */ + if ( ( rc = fdt_path ( &sysfdt, path, offset ) ) != 0 ) { + DBGC ( colour, "HART could not find %s: %s\n", + path, strerror ( rc ) ); + return rc; + } + + return 0; +} + +/** + * Check for supported extension + * + * @v ext Extension name (including leading underscore) + * @ret rc Return status code + */ +int hart_supported ( const char *ext ) { + unsigned int offset; + const char *isa; + const char *tmp; + int rc; + + /* Find boot hart node */ + if ( ( rc = hart_node ( &offset ) ) != 0 ) + return rc; + + /* Get ISA description */ + isa = fdt_string ( &sysfdt, offset, "riscv,isa" ); + if ( ! isa ) { + DBGC ( colour, "HART could not identify ISA\n" ); + return -ENOENT; + } + DBGC ( colour, "HART supports %s\n", isa ); + + /* Check for presence of extension */ + tmp = isa; + while ( ( tmp = strstr ( tmp, ext ) ) != NULL ) { + tmp += strlen ( ext ); + if ( ( *tmp == '\0' ) || ( *tmp == '_' ) ) + return 0; + } + + return -ENOTSUP; +} diff --git a/src/arch/riscv/core/riscv_dma.c b/src/arch/riscv/core/riscv_dma.c new file mode 100644 index 000000000..7b48b1bdd --- /dev/null +++ b/src/arch/riscv/core/riscv_dma.c @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <assert.h> +#include <ipxe/zicbom.h> +#include <ipxe/iomap.h> +#include <ipxe/dma.h> + +/** @file + * + * iPXE DMA API for RISC-V + * + */ + +/** Minimum alignment for coherent DMA allocations + * + * We set this sufficiently high to ensure that we do not end up with + * both cached and uncached uses in the same cacheline. + */ +#define RISCV_DMA_ALIGN 256 + +/** + * Map buffer for DMA + * + * @v dma DMA device + * @v map DMA mapping to fill in + * @v addr Buffer address + * @v len Length of buffer + * @v flags Mapping flags + * @ret rc Return status code + */ +static int riscv_dma_map ( struct dma_device *dma, + struct dma_mapping *map, + void *addr, size_t len, int flags ) { + + /* Sanity check: we cannot support bidirectional mappings */ + assert ( ! ( ( flags & DMA_TX ) & ( flags & DMA_RX ) ) ); + + /* Populate mapping */ + map->dma = dma; + map->offset = 0; + map->token = NULL; + + /* Flush cached data to transmit buffers */ + if ( flags & DMA_TX ) + cache_clean ( addr, len ); + + /* Invalidate cached data in receive buffers and record address */ + if ( flags & DMA_RX ) { + cache_invalidate ( addr, len ); + map->token = addr; + } + + /* Increment mapping count (for debugging) */ + if ( DBG_LOG ) + dma->mapped++; + + return 0; +} + +/** + * Unmap buffer + * + * @v map DMA mapping + * @v len Used length + */ +static void riscv_dma_unmap ( struct dma_mapping *map, size_t len ) { + struct dma_device *dma = map->dma; + void *addr = map->token; + + /* Invalidate cached data in receive buffers */ + if ( addr ) + cache_invalidate ( addr, len ); + + /* Clear mapping */ + map->dma = NULL; + + /* Decrement mapping count (for debugging) */ + if ( DBG_LOG ) + dma->mapped--; +} + +/** + * Allocate and map DMA-coherent buffer + * + * @v dma DMA device + * @v map DMA mapping to fill in + * @v len Length of buffer + * @v align Physical alignment + * @ret addr Buffer address, or NULL on error + */ +static void * riscv_dma_alloc ( struct dma_device *dma, + struct dma_mapping *map, + size_t len, size_t align ) { + physaddr_t phys; + void *addr; + void *caddr; + + /* Round up length and alignment */ + len = ( ( len + RISCV_DMA_ALIGN - 1 ) & ~( RISCV_DMA_ALIGN - 1 ) ); + if ( align < RISCV_DMA_ALIGN ) + align = RISCV_DMA_ALIGN; + + /* Allocate from heap */ + addr = malloc_phys ( len, align ); + if ( ! addr ) + return NULL; + + /* Invalidate any existing cached data */ + cache_invalidate ( addr, len ); + + /* Record mapping */ + map->dma = dma; + map->token = addr; + + /* Calculate coherently-mapped virtual address */ + phys = virt_to_phys ( addr ); + assert ( phys == ( ( uint32_t ) phys ) ); + caddr = ( ( void * ) ( intptr_t ) ( phys + svpage_dma32() ) ); + assert ( phys == virt_to_phys ( caddr ) ); + DBGC ( dma, "DMA allocated [%#08lx,%#08lx) via %p\n", + phys, ( phys + len ), caddr ); + + /* Increment allocation count (for debugging) */ + if ( DBG_LOG ) + dma->allocated++; + + return caddr; +} + +/** + * Unmap and free DMA-coherent buffer + * + * @v dma DMA device + * @v map DMA mapping + * @v addr Buffer address + * @v len Length of buffer + */ +static void riscv_dma_free ( struct dma_mapping *map, + void *addr, size_t len ) { + struct dma_device *dma = map->dma; + + /* Sanity check */ + assert ( virt_to_phys ( addr ) == virt_to_phys ( map->token ) ); + + /* Round up length to match allocation */ + len = ( ( len + RISCV_DMA_ALIGN - 1 ) & ~( RISCV_DMA_ALIGN - 1 ) ); + + /* Free original allocation */ + free_phys ( map->token, len ); + + /* Clear mapping */ + map->dma = NULL; + map->token = NULL; + + /* Decrement allocation count (for debugging) */ + if ( DBG_LOG ) + dma->allocated--; +} + +PROVIDE_DMAAPI ( riscv, dma_map, riscv_dma_map ); +PROVIDE_DMAAPI ( riscv, dma_unmap, riscv_dma_unmap ); +PROVIDE_DMAAPI ( riscv, dma_alloc, riscv_dma_alloc ); +PROVIDE_DMAAPI ( riscv, dma_free, riscv_dma_free ); +PROVIDE_DMAAPI ( riscv, dma_umalloc, riscv_dma_alloc ); +PROVIDE_DMAAPI ( riscv, dma_ufree, riscv_dma_free ); +PROVIDE_DMAAPI_INLINE ( riscv, dma_set_mask ); +PROVIDE_DMAAPI_INLINE ( riscv, dma ); diff --git a/src/arch/arm/interface/efi/efiarm_nap.c b/src/arch/riscv/core/riscv_io.c index fba7a5d82..756b39752 100644 --- a/src/arch/arm/interface/efi/efiarm_nap.c +++ b/src/arch/riscv/core/riscv_io.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2016 Michael Brown <mbrown@fensystems.co.uk>. + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -23,35 +23,24 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); -#include <ipxe/nap.h> -#include <ipxe/efi/efi.h> +#include <ipxe/io.h> +#include <ipxe/riscv_io.h> /** @file * - * iPXE CPU sleeping API for EFI + * iPXE I/O API for RISC-V * */ -/** - * Sleep until next interrupt - * - */ -static void efiarm_cpu_nap ( void ) { - /* - * I can't find any EFI API that allows us to put the CPU to - * sleep. The CpuSleep() function is defined in CpuLib.h, but - * isn't part of any exposed protocol so we have no way to - * call it. - * - * The EFI shell doesn't seem to bother sleeping the CPU; it - * just sits there idly burning power. - * - * If a shutdown is in progess, there may be nothing to - * generate an interrupt since the timer is disabled in the - * first step of ExitBootServices(). - */ - if ( ! efi_shutdown_in_progress ) - __asm__ __volatile__ ( "wfi" ); -} - -PROVIDE_NAP ( efiarm, cpu_nap, efiarm_cpu_nap ); +PROVIDE_IOAPI_INLINE ( riscv, phys_to_bus ); +PROVIDE_IOAPI_INLINE ( riscv, bus_to_phys ); +PROVIDE_IOAPI_INLINE ( riscv, readb ); +PROVIDE_IOAPI_INLINE ( riscv, readw ); +PROVIDE_IOAPI_INLINE ( riscv, readl ); +PROVIDE_IOAPI_INLINE ( riscv, writeb ); +PROVIDE_IOAPI_INLINE ( riscv, writew ); +PROVIDE_IOAPI_INLINE ( riscv, writel ); +PROVIDE_IOAPI_INLINE ( riscv, readq ); +PROVIDE_IOAPI_INLINE ( riscv, writeq ); +PROVIDE_IOAPI_INLINE ( riscv, mb ); +PROVIDE_DUMMY_PIO ( riscv ); diff --git a/src/arch/riscv/core/riscv_string.c b/src/arch/riscv/core/riscv_string.c new file mode 100644 index 000000000..e28dc8951 --- /dev/null +++ b/src/arch/riscv/core/riscv_string.c @@ -0,0 +1,233 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +/** @file + * + * Optimised string operations + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <string.h> + +/** + * Copy memory area + * + * @v dest Destination address + * @v src Source address + * @v len Length + * @ret dest Destination address + */ +void riscv_memcpy ( void *dest, const void *src, size_t len ) { + size_t len_pre; + size_t len_mid; + size_t len_post; + unsigned long discard_data; + + /* Calculate pre-aligned, aligned, and post-aligned lengths. + * (Align on the destination address, on the assumption that + * misaligned stores are likely to be more expensive than + * misaligned loads.) + */ + len_pre = ( ( sizeof ( unsigned long ) - ( ( intptr_t ) dest ) ) & + ( sizeof ( unsigned long ) - 1 ) ); + if ( len_pre > len ) + len_pre = len; + len -= len_pre; + len_mid = ( len & ~( sizeof ( unsigned long ) - 1 ) ); + len -= len_mid; + len_post = len; + + /* Copy pre-aligned section */ + __asm__ __volatile__ ( "j 2f\n\t" + "\n1:\n\t" + "lb %2, (%1)\n\t" + "sb %2, (%0)\n\t" + "addi %0, %0, 1\n\t" + "addi %1, %1, 1\n\t" + "\n2:\n\t" + "bne %0, %3, 1b\n\t" + : "+r" ( dest ), "+r" ( src ), + "=&r" ( discard_data ) + : "r" ( dest + len_pre ) + : "memory" ); + + /* Copy aligned section */ + __asm__ __volatile__ ( "j 2f\n\t" + "\n1:\n\t" + LOADN " %2, (%1)\n\t" + STOREN " %2, (%0)\n\t" + "addi %0, %0, %4\n\t" + "addi %1, %1, %4\n\t" + "\n2:\n\t" + "bne %0, %3, 1b\n\t" + : "+r" ( dest ), "+r" ( src ), + "=&r" ( discard_data ) + : "r" ( dest + len_mid ), + "i" ( sizeof ( unsigned long ) ) + : "memory" ); + + /* Copy post-aligned section */ + __asm__ __volatile__ ( "j 2f\n\t" + "\n1:\n\t" + "lb %2, (%1)\n\t" + "sb %2, (%0)\n\t" + "addi %0, %0, 1\n\t" + "addi %1, %1, 1\n\t" + "\n2:\n\t" + "bne %0, %3, 1b\n\t" + : "+r" ( dest ), "+r" ( src ), + "=&r" ( discard_data ) + : "r" ( dest + len_post ) + : "memory" ); +} + +/** + * Zero memory region + * + * @v dest Destination region + * @v len Length + */ +void riscv_bzero ( void *dest, size_t len ) { + size_t len_pre; + size_t len_mid; + size_t len_post; + + /* Calculate pre-aligned, aligned, and post-aligned lengths */ + len_pre = ( ( sizeof ( unsigned long ) - ( ( intptr_t ) dest ) ) & + ( sizeof ( unsigned long ) - 1 ) ); + if ( len_pre > len ) + len_pre = len; + len -= len_pre; + len_mid = ( len & ~( sizeof ( unsigned long ) - 1 ) ); + len -= len_mid; + len_post = len; + + /* Zero pre-aligned section */ + __asm__ __volatile__ ( "j 2f\n\t" + "\n1:\n\t" + "sb zero, (%0)\n\t" + "addi %0, %0, 1\n\t" + "\n2:\n\t" + "bne %0, %1, 1b\n\t" + : "+r" ( dest ) + : "r" ( dest + len_pre ) + : "memory" ); + + /* Zero aligned section */ + __asm__ __volatile__ ( "j 2f\n\t" + "\n1:\n\t" + STOREN " zero, (%0)\n\t" + "addi %0, %0, %2\n\t" + "\n2:\n\t" + "bne %0, %1, 1b\n\t" + : "+r" ( dest ) + : "r" ( dest + len_mid ), + "i" ( sizeof ( unsigned long ) ) + : "memory" ); + + /* Zero post-aligned section */ + __asm__ __volatile__ ( "j 2f\n\t" + "\n1:\n\t" + "sb zero, (%0)\n\t" + "addi %0, %0, 1\n\t" + "\n2:\n\t" + "bne %0, %1, 1b\n\t" + : "+r" ( dest ) + : "r" ( dest + len_post ) + : "memory" ); +} + +/** + * Fill memory region + * + * @v dest Destination region + * @v len Length + * @v character Fill character + * + * The unusual parameter order is to allow for more efficient + * tail-calling to riscv_bzero() when zeroing a region. + */ +void riscv_memset ( void *dest, size_t len, int character ) { + + /* Do nothing if length is zero */ + if ( ! len ) + return; + + /* Use optimised zeroing code if applicable */ + if ( character == 0 ) { + riscv_bzero ( dest, len ); + return; + } + + /* Fill one byte at a time. Calling memset() with a non-zero + * value is relatively rare and unlikely to be + * performance-critical. + */ + __asm__ __volatile__ ( "\n1:\n\t" + "sb %2, (%0)\n\t" + "addi %0, %0, 1\n\t" + "bne %0, %1, 1b\n\t" + : "+r" ( dest ) + : "r" ( dest + len ), "r" ( character ) + : "memory" ); +} + +/** + * Copy (possibly overlapping) memory region + * + * @v dest Destination region + * @v src Source region + * @v len Length + */ +void riscv_memmove ( void *dest, const void *src, size_t len ) { + void *orig_dest = dest; + unsigned long discard_data; + + /* Do nothing if length is zero */ + if ( ! len ) + return; + + /* Use memcpy() if copy direction is forwards */ + if ( dest <= src ) { + memcpy ( dest, src, len ); + return; + } + + /* Assume memmove() is not performance-critical, and perform a + * bytewise copy backwards for simplicity. + */ + dest += len; + src += len; + __asm__ __volatile__ ( "\n1:\n\t" + "addi %1, %1, -1\n\t" + "addi %0, %0, -1\n\t" + "lb %2, (%1)\n\t" + "sb %2, (%0)\n\t" + "bne %0, %3, 1b\n\t" + : "+r" ( dest ), "+r" ( src ), + "=&r" ( discard_data ) + : "r" ( orig_dest ) + : "memory" ); +} diff --git a/src/arch/x86/interface/efi/efix86_nap.c b/src/arch/riscv/core/riscv_strings.S index 296876b85..eb1b397b9 100644 --- a/src/arch/x86/interface/efi/efix86_nap.c +++ b/src/arch/riscv/core/riscv_strings.S @@ -1,5 +1,5 @@ /* - * Copyright (C) 2008 Michael Brown <mbrown@fensystems.co.uk>. + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -21,37 +21,50 @@ * COPYING.UBDL), provided that you have satisfied its requirements. */ -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <ipxe/nap.h> -#include <ipxe/efi/efi.h> + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) /** @file * - * iPXE CPU sleeping API for EFI + * Byte swapping * */ + .section ".note.GNU-stack", "", @progbits + .text + /** - * Sleep until next interrupt + * Find first (i.e. least significant) set bit * + * @v value Value + * @ret lsb Least significant bit set in value (LSB=1), or zero */ -static void efix86_cpu_nap ( void ) { - /* - * I can't find any EFI API that allows us to put the CPU to - * sleep. The CpuSleep() function is defined in CpuLib.h, but - * isn't part of any exposed protocol so we have no way to - * call it. - * - * The EFI shell doesn't seem to bother sleeping the CPU; it - * just sits there idly burning power. - * - * If a shutdown is in progess, there may be nothing to - * generate an interrupt since the timer is disabled in the - * first step of ExitBootServices(). - */ - if ( ! efi_shutdown_in_progress ) - __asm__ __volatile__ ( "hlt" ); -} + .section ".text.riscv_ffs" + .globl riscv_ffs +riscv_ffs: + beqz a0, 2f + mv t0, a0 + li a0, ( __riscv_xlen + 1 ) +1: slli t0, t0, 1 + addi a0, a0, -1 + bnez t0, 1b +2: ret + .size riscv_ffs, . - riscv_ffs -PROVIDE_NAP ( efix86, cpu_nap, efix86_cpu_nap ); +/** + * Find last (i.e. most significant) set bit + * + * @v value Value + * @ret msb Most significant bit set in value (LSB=1), or zero + */ + .section ".text.riscv_fls" + .globl riscv_fls +riscv_fls: + beqz a0, 2f + mv t0, a0 + li a0, __riscv_xlen + bltz t0, 2f +1: slli t0, t0, 1 + addi a0, a0, -1 + bgez t0, 1b +2: ret + .size riscv_fls, . - riscv_fls diff --git a/src/arch/riscv/core/riscv_tcpip.S b/src/arch/riscv/core/riscv_tcpip.S new file mode 100644 index 000000000..8c3e90ba6 --- /dev/null +++ b/src/arch/riscv/core/riscv_tcpip.S @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * TCP/IP checksum + * + */ + + .section ".note.GNU-stack", "", @progbits + .text + +/** + * Calculate continued TCP/IP checkum + * + * @v partial Checksum of already-summed data, in network byte order + * @v data Data buffer + * @v len Length of data buffer + * @ret cksum Updated checksum, in network byte order + * + * In practice, this routine will only ever be called with a data + * pointer aligned to a 16-bit boundary. We optimise for this case, + * ensuring that the code would still give correct output if called + * with a misaligned pointer. + */ + .section ".text.tcpip_continue_chksum", "ax", @progbits + .globl tcpip_continue_chksum +tcpip_continue_chksum: + + /* Set up register usage: + * + * a0: checksum low xlen bits + * a1: data pointer + * a2: end of data pointer + * a3: end of data pointer minus a constant offset of interest + * a4: checksum high bits (guaranteed to never carry) / constant 0xffff + * a5: temporary register + */ + not a0, a0 + add a2, a2, a1 + addi a3, a2, -( __riscv_xlen / 8 ) + mv a4, zero + + /* Skip aligned checksumming if data is too short */ + bgtu a1, a3, post_aligned + + /* Checksum 16-bit words until we reach xlen-bit alignment (or + * one byte past xlen-bit alignment). + */ + j 2f +1: lhu a5, (a1) + addi a1, a1, 2 + add a4, a4, a5 +2: andi a5, a1, ( ( ( __riscv_xlen / 8 ) - 1 ) & ~1 ) + bnez a5, 1b + + /* Checksum aligned xlen-bit words */ + j 2f +1: LOADN a5, (a1) + addi a1, a1, ( __riscv_xlen / 8 ) + add a0, a0, a5 + sltu a5, a0, a5 + add a4, a4, a5 +2: bleu a1, a3, 1b + +post_aligned: + /* Checksum remaining 16-bit words */ + addi a3, a2, -2 + j 2f +1: lhu a5, (a1) + addi a1, a1, 2 + add a4, a4, a5 +2: bleu a1, a3, 1b + + /* Checksum final byte if present */ + beq a1, a2, 1f + lbu a5, (a1) + add a4, a4, a5 +1: + /* Fold down to xlen+1 bits */ + add a0, a0, a4 + sltu a4, a0, a4 + + /* Fold down to (xlen/2)+2 bits */ + slli a5, a0, ( __riscv_xlen / 2 ) + srli a0, a0, ( __riscv_xlen / 2 ) + srli a5, a5, ( __riscv_xlen / 2 ) + add a0, a0, a4 + add a0, a0, a5 + + /* Load constant 0xffff for use in subsequent folding */ + li a4, 0xffff + +#if __riscv_xlen >= 64 + /* Fold down to (xlen/4)+3 bits (if xlen >= 64) */ + and a5, a0, a4 + srli a0, a0, ( __riscv_xlen / 4 ) + add a0, a0, a5 +#endif + + /* Fold down to 16+1 bits */ + and a5, a0, a4 + srli a0, a0, 16 + add a0, a0, a5 + + /* Fold down to 16 bits */ + srli a5, a0, 16 + add a0, a0, a5 + srli a5, a0, 17 + add a0, a0, a5 + and a0, a0, a4 + + /* Negate and return */ + xor a0, a0, a4 + ret + .size tcpip_continue_chksum, . - tcpip_continue_chksum diff --git a/src/arch/riscv/core/setjmp.S b/src/arch/riscv/core/setjmp.S new file mode 100644 index 000000000..69d844ac9 --- /dev/null +++ b/src/arch/riscv/core/setjmp.S @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * Long jumps + * + */ + + .section ".note.GNU-stack", "", @progbits + .text + + /* Must match jmp_buf structure layout */ + .struct 0 +env_ra: .space ( __riscv_xlen / 8 ) +env_sp: .space ( __riscv_xlen / 8 ) +env_s0: .space ( __riscv_xlen / 8 ) +env_s1: .space ( __riscv_xlen / 8 ) +env_s2: .space ( __riscv_xlen / 8 ) +env_s3: .space ( __riscv_xlen / 8 ) +env_s4: .space ( __riscv_xlen / 8 ) +env_s5: .space ( __riscv_xlen / 8 ) +env_s6: .space ( __riscv_xlen / 8 ) +env_s7: .space ( __riscv_xlen / 8 ) +env_s8: .space ( __riscv_xlen / 8 ) +env_s9: .space ( __riscv_xlen / 8 ) +env_s10: .space ( __riscv_xlen / 8 ) +env_s11: .space ( __riscv_xlen / 8 ) + .previous + +/* + * Save stack context for non-local goto + */ + .section ".text.setjmp", "ax", @progbits + .globl setjmp +setjmp: + /* Save registers */ + STOREN ra, env_ra(a0) + STOREN sp, env_sp(a0) + STOREN s0, env_s0(a0) + STOREN s1, env_s1(a0) + STOREN s2, env_s2(a0) + STOREN s3, env_s3(a0) + STOREN s4, env_s4(a0) + STOREN s5, env_s5(a0) + STOREN s6, env_s6(a0) + STOREN s7, env_s7(a0) + STOREN s8, env_s8(a0) + STOREN s9, env_s9(a0) + STOREN s10, env_s10(a0) + STOREN s11, env_s11(a0) + /* Return zero when returning as setjmp() */ + mv a0, zero + ret + .size setjmp, . - setjmp + +/* + * Non-local jump to a saved stack context + */ + .section ".text.longjmp", "ax", @progbits + .globl longjmp +longjmp: + /* Restore registers */ + LOADN s11, env_s11(a0) + LOADN s10, env_s10(a0) + LOADN s9, env_s9(a0) + LOADN s8, env_s8(a0) + LOADN s7, env_s7(a0) + LOADN s6, env_s6(a0) + LOADN s5, env_s5(a0) + LOADN s4, env_s4(a0) + LOADN s3, env_s3(a0) + LOADN s2, env_s2(a0) + LOADN s1, env_s1(a0) + LOADN s0, env_s0(a0) + LOADN sp, env_sp(a0) + LOADN ra, env_ra(a0) + /* Force result to non-zero */ + seqz a0, a1 + or a0, a0, a1 + /* Return to setjmp() caller */ + ret + .size longjmp, . - longjmp diff --git a/src/arch/riscv/core/stack.S b/src/arch/riscv/core/stack.S new file mode 100644 index 000000000..1cd1da7c5 --- /dev/null +++ b/src/arch/riscv/core/stack.S @@ -0,0 +1,45 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * Internal stack + * + */ + + .section ".note.GNU-stack", "", @progbits + .text + +#define STACK_ALIGN 16 + +#define STACK_SIZE 8192 + + .section ".stack", "aw", @nobits + .balign STACK_ALIGN + .globl _stack +_stack: + .space STACK_SIZE + .globl _estack +_estack: diff --git a/src/arch/riscv/core/svpage.c b/src/arch/riscv/core/svpage.c new file mode 100644 index 000000000..e25e3920a --- /dev/null +++ b/src/arch/riscv/core/svpage.c @@ -0,0 +1,292 @@ +/* + * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> +#include <strings.h> +#include <assert.h> +#include <ipxe/hart.h> +#include <ipxe/iomap.h> + +/** @file + * + * Supervisor page table management + * + * With the 64-bit paging schemes (Sv39, Sv48, and Sv57) we choose to + * identity-map as much as possible of the physical address space via + * PTEs 0-255, and place a recursive page table entry in PTE 511 which + * allows PTEs 256-510 to be used to map 1GB "gigapages" within the + * top 256GB of the 64-bit address space. At least one of these PTEs + * will already be in use to map iPXE itself. The remaining PTEs may + * be used to map I/O devices. + */ + +/** A page table */ +struct page_table { + /** Page table entry */ + uint64_t pte[512]; +}; + +/** Page table entry flags */ +enum pte_flags { + /** Page table entry is valid */ + PTE_V = 0x01, + /** Page is readable */ + PTE_R = 0x02, + /** Page is writable */ + PTE_W = 0x04, + /** Page has been accessed */ + PTE_A = 0x40, + /** Page is dirty */ + PTE_D = 0x80, + /** Page is the last page in an allocation + * + * This bit is ignored by the hardware. We use it to track + * the size of allocations made by ioremap(). + */ + PTE_LAST = 0x100, +}; + +/** Page-based memory type (Svpbmt) */ +#define PTE_SVPBMT( x ) ( ( ( unsigned long long ) (x) ) << 61 ) + +/** Page is non-cacheable memory (Svpbmt) */ +#define PTE_SVPBMT_NC PTE_SVPBMT ( 1 ) + +/** Page maps I/O addresses (Svpbmt) */ +#define PTE_SVPBMT_IO PTE_SVPBMT ( 2 ) + +/** Page table entry address */ +#define PTE_PPN( addr ) ( (addr) >> 2 ) + +/** The page table */ +extern struct page_table page_table; + +/** Maximum number of I/O pages */ +#define MAP_PAGE_COUNT \ + ( sizeof ( page_table.pte ) / sizeof ( page_table.pte[0] ) ) + +/** I/O page size + * + * We choose to use 1GB "gigapages", since these are supported by all + * paging levels. + */ +#define MAP_PAGE_SIZE 0x40000000UL + +/** I/O page base address + * + * The recursive page table entry maps the high 512GB of the 64-bit + * address space as 1GB "gigapages". + */ +#define MAP_BASE ( ( void * ) ( intptr_t ) ( -1ULL << 39 ) ) + +/** Coherent DMA mapping of the 32-bit address space */ +static void *svpage_dma32_base; + +/** Size of the coherent DMA mapping */ +#define DMA32_LEN ( ( size_t ) 0x100000000ULL ) + +/** + * Map pages + * + * @v phys Physical address + * @v len Length + * @v attrs Page attributes + * @ret virt Mapped virtual address, or NULL on error + */ +static void * svpage_map ( physaddr_t phys, size_t len, unsigned long attrs ) { + unsigned long satp; + unsigned long start; + unsigned int count; + unsigned int stride; + unsigned int first; + unsigned int i; + size_t offset; + void *virt; + + DBGC ( &page_table, "SVPAGE mapping %#08lx+%#zx attrs %#016lx\n", + phys, len, attrs ); + + /* Sanity checks */ + if ( ! len ) + return NULL; + assert ( attrs & PTE_V ); + + /* Use physical address directly if paging is disabled */ + __asm__ ( "csrr %0, satp" : "=r" ( satp ) ); + if ( ! satp ) { + virt = phys_to_virt ( phys ); + DBGC ( &page_table, "SVPAGE mapped %#08lx+%#zx to %p (no " + "paging)\n", phys, len, virt ); + return virt; + } + + /* Round down start address to a page boundary */ + start = ( phys & ~( MAP_PAGE_SIZE - 1 ) ); + offset = ( phys - start ); + assert ( offset < MAP_PAGE_SIZE ); + + /* Calculate number of pages required */ + count = ( ( offset + len + MAP_PAGE_SIZE - 1 ) / MAP_PAGE_SIZE ); + assert ( count != 0 ); + assert ( count <= MAP_PAGE_COUNT ); + + /* Round up number of pages to a power of two */ + stride = ( 1 << fls ( count - 1 ) ); + assert ( count <= stride ); + + /* Allocate pages */ + for ( first = 0 ; first < MAP_PAGE_COUNT ; first += stride ) { + + /* Calculate virtual address */ + virt = ( MAP_BASE + ( first * MAP_PAGE_SIZE ) + offset ); + + /* Check that page table entries are available */ + for ( i = first ; i < ( first + count ) ; i++ ) { + if ( page_table.pte[i] & PTE_V ) { + virt = NULL; + break; + } + } + if ( ! virt ) + continue; + + /* Create page table entries */ + for ( i = first ; i < ( first + count ) ; i++ ) { + page_table.pte[i] = ( PTE_PPN ( start ) | attrs ); + start += MAP_PAGE_SIZE; + } + + /* Mark last page as being the last in this allocation */ + page_table.pte[ i - 1 ] |= PTE_LAST; + + /* Synchronise page table updates */ + __asm__ __volatile__ ( "sfence.vma" ); + + /* Return virtual address */ + DBGC ( &page_table, "SVPAGE mapped %#08lx+%#zx to %p using " + "PTEs [%d-%d]\n", phys, len, virt, first, + ( first + count - 1 ) ); + return virt; + } + + DBGC ( &page_table, "SVPAGE could not map %#08lx+%#zx\n", + phys, len ); + return NULL; +} + +/** + * Unmap pages + * + * @v virt Virtual address + */ +static void svpage_unmap ( const volatile void *virt ) { + unsigned long satp; + unsigned int first; + unsigned int i; + int is_last; + + DBGC ( &page_table, "SVPAGE unmapping %p\n", virt ); + + /* Do nothing if paging is disabled */ + __asm__ ( "csrr %0, satp" : "=r" ( satp ) ); + if ( ! satp ) + return; + + /* Calculate first page table entry */ + first = ( ( virt - MAP_BASE ) / MAP_PAGE_SIZE ); + + /* Ignore unmappings outside of the I/O range */ + if ( first >= MAP_PAGE_COUNT ) + return; + + /* Clear page table entries */ + for ( i = first ; ; i++ ) { + + /* Sanity check */ + assert ( page_table.pte[i] & PTE_V ); + + /* Check if this is the last page in this allocation */ + is_last = ( page_table.pte[i] & PTE_LAST ); + + /* Clear page table entry */ + page_table.pte[i] = 0; + + /* Terminate if this was the last page */ + if ( is_last ) + break; + } + + /* Synchronise page table updates */ + __asm__ __volatile__ ( "sfence.vma" ); + + DBGC ( &page_table, "SVPAGE unmapped %p using PTEs [%d-%d]\n", + virt, first, i ); +} + +/** + * Map pages for I/O + * + * @v bus_addr Bus address + * @v len Length of region + * @ret io_addr I/O address + */ +static void * svpage_ioremap ( unsigned long bus_addr, size_t len ) { + unsigned long attrs = ( PTE_V | PTE_R | PTE_W | PTE_A | PTE_D ); + int rc; + + /* Add Svpbmt attributes if applicable */ + if ( ( rc = hart_supported ( "_svpbmt" ) ) == 0 ) + attrs |= PTE_SVPBMT_IO; + + /* Map pages for I/O */ + return svpage_map ( bus_addr, len, attrs ); +} + +/** + * Get 32-bit address space coherent DMA mapping address + * + * @ret base Coherent DMA mapping base address + */ +void * svpage_dma32 ( void ) { + unsigned long attrs = ( PTE_V | PTE_R | PTE_W | PTE_A | PTE_D ); + int rc; + + /* Add Svpbmt attributes if applicable */ + if ( ( rc = hart_supported ( "_svpbmt" ) ) == 0 ) + attrs |= PTE_SVPBMT_NC; + + /* Create mapping, if necessary */ + if ( ! svpage_dma32_base ) + svpage_dma32_base = svpage_map ( 0, DMA32_LEN, attrs ); + + /* Sanity check */ + assert ( virt_to_phys ( svpage_dma32_base ) == 0 ); + + return svpage_dma32_base; +} + +PROVIDE_IOMAP_INLINE ( svpage, io_to_bus ); +PROVIDE_IOMAP ( svpage, ioremap, svpage_ioremap ); +PROVIDE_IOMAP ( svpage, iounmap, svpage_unmap ); diff --git a/src/arch/loong64/interface/efi/efiloong64_nap.c b/src/arch/riscv/core/xthead.c index 0a7609783..c947c9dc8 100644 --- a/src/arch/loong64/interface/efi/efiloong64_nap.c +++ b/src/arch/riscv/core/xthead.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023, Xiaotian Wu <wuxiaotian@loongson.cn> + * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -23,35 +23,43 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); -#include <ipxe/nap.h> -#include <ipxe/efi/efi.h> - /** @file * - * iPXE CPU sleeping API for EFI + * T-Head vendor extensions * */ +#include <ipxe/sbi.h> +#include <ipxe/xthead.h> + +/** Colour for debug messages */ +#define colour THEAD_MVENDORID + /** - * Sleep until next interrupt + * Check for a T-Head feature via SXSTATUS register * + * @v feature Feature bit + * @ret supported Feature is supported */ -static void efiloong64_cpu_nap ( void ) { - /* - * I can't find any EFI API that allows us to put the CPU to - * sleep. The CpuSleep() function is defined in CpuLib.h, but - * isn't part of any exposed protocol so we have no way to - * call it. - * - * The EFI shell doesn't seem to bother sleeping the CPU; it - * just sits there idly burning power. - * - * If a shutdown is in progess, there may be nothing to - * generate an interrupt since the timer is disabled in the - * first step of ExitBootServices(). - */ - if ( ! efi_shutdown_in_progress ) - __asm__ __volatile__ ( "idle 0" ); -} +int xthead_supported ( unsigned long feature ) { + struct sbi_return ret; + unsigned long sxstatus; -PROVIDE_NAP ( efiloong64, cpu_nap, efiloong64_cpu_nap ); + /* Check for a T-Head CPU */ + ret = sbi_ecall_0 ( SBI_BASE, SBI_BASE_MVENDORID ); + if ( ret.error ) + return 0; + if ( ret.value != THEAD_MVENDORID ) { + DBGC ( colour, "THEAD vendor ID mismatch: %#08lx\n", + ret.value ); + return 0; + } + + /* Read SXSTATUS CSR */ + __asm__ ( "csrr %0, %1" + : "=r" ( sxstatus ) : "i" ( THEAD_SXSTATUS ) ); + DBGC ( colour, "THEAD sxstatus %#08lx\n", sxstatus ); + + /* Check feature bit */ + return ( !! ( sxstatus & feature ) ); +} diff --git a/src/arch/riscv/core/zicbom.c b/src/arch/riscv/core/zicbom.c new file mode 100644 index 000000000..28ff62c22 --- /dev/null +++ b/src/arch/riscv/core/zicbom.c @@ -0,0 +1,255 @@ +/* + * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** @file + * + * Cache-block management operations (Zicbom) + * + * We support explicit cache management operations on I/O buffers. + * These are guaranteed to be aligned on their own size and at least + * as large as a (reasonable) cacheline, and therefore cannot cross a + * cacheline boundary. + */ + +#include <stdint.h> +#include <ipxe/hart.h> +#include <ipxe/xthead.h> +#include <ipxe/zicbom.h> + +/** Minimum supported cacheline size + * + * We assume that cache management operations will ignore the least + * significant address bits, and so we are safe to assume a cacheline + * size that is smaller than the size actually used by the CPU. + * + * Cache clean and invalidate loops could be made faster by detecting + * the actual cacheline size. + */ +#define CACHE_STRIDE 32 + +/** A cache management extension */ +struct cache_extension { + /** + * Clean data cache (i.e. write cached content back to memory) + * + * @v first First byte + * @v last Last byte + */ + void ( * clean ) ( const void *first, const void *last ); + /** + * Invalidate data cache (i.e. discard any cached content) + * + * @v first First byte + * @v last Last byte + */ + void ( * invalidate ) ( void *first, void *last ); +}; + +/** Define an operation to clean the data cache */ +#define CACHE_CLEAN( extension, insn ) \ + static void extension ## _clean ( const void *first, \ + const void *last ) { \ + \ + __asm__ __volatile__ ( ".option arch, +" #extension "\n\t" \ + "\n1:\n\t" \ + insn "\n\t" \ + "addi %0, %0, %2\n\t" \ + "bltu %0, %1, 1b\n\t" \ + : "+r" ( first ) \ + : "r" ( last ), "i" ( CACHE_STRIDE ) ); \ + } + +/** Define an operation to invalidate the data cache */ +#define CACHE_INVALIDATE( extension, insn ) \ + static void extension ## _invalidate ( void *first, \ + void *last ) { \ + \ + __asm__ __volatile__ ( ".option arch, +" #extension "\n\t" \ + "\n1:\n\t" \ + insn "\n\t" \ + "addi %0, %0, %2\n\t" \ + "bltu %0, %1, 1b\n\t" \ + : "+r" ( first ) \ + : "r" ( last ), "i" ( CACHE_STRIDE ) \ + : "memory" ); \ + } + +/** Define a cache management extension */ +#define CACHE_EXTENSION( extension, clean_insn, invalidate_insn ) \ + CACHE_CLEAN ( extension, clean_insn ); \ + CACHE_INVALIDATE ( extension, invalidate_insn ); \ + static struct cache_extension extension = { \ + .clean = extension ## _clean, \ + .invalidate = extension ## _invalidate, \ + }; + +/** The standard Zicbom extension */ +CACHE_EXTENSION ( zicbom, "cbo.clean (%0)", "cbo.inval (%0)" ); + +/** The T-Head cache management extension */ +CACHE_EXTENSION ( xtheadcmo, "th.dcache.cva %0", "th.dcache.iva %0" ); + +/** + * Clean data cache (with fully coherent memory) + * + * @v first First byte + * @v last Last byte + */ +static void cache_coherent_clean ( const void *first __unused, + const void *last __unused ) { + /* Nothing to do */ +} + +/** + * Invalidate data cache (with fully coherent memory) + * + * @v first First byte + * @v last Last byte + */ +static void cache_coherent_invalidate ( void *first __unused, + void *last __unused ) { + /* Nothing to do */ +} + +/** Dummy cache management extension for fully coherent memory */ +static struct cache_extension cache_coherent = { + .clean = cache_coherent_clean, + .invalidate = cache_coherent_invalidate, +}; + +static void cache_auto_detect ( void ); +static void cache_auto_clean ( const void *first, const void *last ); +static void cache_auto_invalidate ( void *first, void *last ); + +/** The autodetect cache management extension */ +static struct cache_extension cache_auto = { + .clean = cache_auto_clean, + .invalidate = cache_auto_invalidate, +}; + +/** Active cache management extension */ +static struct cache_extension *cache_extension = &cache_auto; + +/** + * Clean data cache (i.e. write cached content back to memory) + * + * @v start Start address + * @v len Length + */ +void cache_clean ( const void *start, size_t len ) { + const void *first; + const void *last; + + /* Do nothing for zero-length buffers */ + if ( ! len ) + return; + + /* Construct address range */ + first = ( ( const void * ) + ( ( ( intptr_t ) start ) & ~( CACHE_STRIDE - 1 ) ) ); + last = ( start + len - 1 ); + + /* Clean cache lines */ + cache_extension->clean ( first, last ); +} + +/** + * Invalidate data cache (i.e. discard any cached content) + * + * @v start Start address + * @v len Length + */ +void cache_invalidate ( void *start, size_t len ) { + void *first; + void *last; + + /* Do nothing for zero-length buffers */ + if ( ! len ) + return; + + /* Construct address range */ + first = ( ( void * ) + ( ( ( intptr_t ) start ) & ~( CACHE_STRIDE - 1 ) ) ); + last = ( start + len - 1 ); + + /* Invalidate cache lines */ + cache_extension->invalidate ( first, last ); +} + +/** + * Autodetect and clean data cache + * + * @v first First byte + * @v last Last byte + */ +static void cache_auto_clean ( const void *first, const void *last ) { + + /* Detect cache extension */ + cache_auto_detect(); + + /* Clean data cache */ + cache_extension->clean ( first, last ); +} + +/** + * Autodetect and invalidate data cache + * + * @v first First byte + * @v last Last byte + */ +static void cache_auto_invalidate ( void *first, void *last ) { + + /* Detect cache extension */ + cache_auto_detect(); + + /* Clean data cache */ + cache_extension->invalidate ( first, last ); +} + +/** + * Autodetect cache + * + */ +static void cache_auto_detect ( void ) { + int rc; + + /* Check for standard Zicbom extension */ + if ( ( rc = hart_supported ( "_zicbom" ) ) == 0 ) { + DBGC ( &cache_extension, "CACHE detected Zicbom\n" ); + cache_extension = &zicbom; + return; + } + + /* Check for T-Head cache management extension */ + if ( xthead_supported ( THEAD_SXSTATUS_THEADISAEE ) ) { + DBGC ( &cache_extension, "CACHE detected XTheadCmo\n" ); + cache_extension = &xtheadcmo; + return; + } + + /* Assume coherent memory if no supported extension detected */ + DBGC ( &cache_extension, "CACHE assuming coherent memory\n" ); + cache_extension = &cache_coherent; +} diff --git a/src/arch/riscv/core/zicntr.c b/src/arch/riscv/core/zicntr.c new file mode 100644 index 000000000..fb632b8c0 --- /dev/null +++ b/src/arch/riscv/core/zicntr.c @@ -0,0 +1,194 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** @file + * + * Base counters and timers extension (Zicntr) + * + */ + +#include <string.h> +#include <errno.h> +#include <ipxe/fdt.h> +#include <ipxe/csr.h> +#include <ipxe/timer.h> + +/** Timer increment per microsecond */ +static unsigned long zicntr_mhz; + +/** Minimum resolution for scaled timer */ +#define ZICNTR_SCALED_HZ 32 + +/** + * Timer scale (expressed as a bit shift) + * + * We use this to avoid the need for 64-bit divsion on 32-bit systems. + */ +static unsigned int zicntr_scale; + +/** Number of timer ticks per scaled timer increment */ +static unsigned long zicntr_ticks; + +/** Colour for debug messages */ +#define colour &zicntr_mhz + +/** + * Get low XLEN bits of current time + * + * @ret time Current time + */ +static inline __attribute__ (( always_inline )) unsigned long +rdtime_low ( void ) { + unsigned long time; + + /* Read low XLEN bits of current time */ + __asm__ __volatile__ ( "rdtime %0" : "=r" ( time ) ); + return time; +} + +/** + * Get current time, scaled to avoid rollover within a realistic timescale + * + * @ret time Scaled current time + */ +static inline __attribute__ (( always_inline )) unsigned long +rdtime_scaled ( void ) { + union { + uint64_t time; + struct { + uint32_t low; + uint32_t high; + }; + } u; + unsigned long tmp __attribute__ (( unused )); + + /* Read full current time */ +#if __riscv_xlen >= 64 + __asm__ __volatile__ ( "rdtime %0" : "=r" ( u.time ) ); +#else + __asm__ __volatile__ ( "1:\n\t" + "rdtimeh %1\n\t" + "rdtime %0\n\t" + "rdtimeh %2\n\t" + "bne %1, %2, 1b\n\t" + : "=r" ( u.low ), "=r" ( u.high ), + "=r" ( tmp ) ); +#endif + + /* Scale time to avoid XLEN-bit rollover */ + return ( u.time >> zicntr_scale ); +} + +/** + * Get current system time in ticks + * + * @ret ticks Current time, in ticks + */ +static unsigned long zicntr_currticks ( void ) { + unsigned long scaled; + + /* Get scaled time and convert to ticks */ + scaled = rdtime_scaled(); + return ( scaled * zicntr_ticks ); +} + +/** + * Delay for a fixed number of microseconds + * + * @v usecs Number of microseconds for which to delay + */ +static void zicntr_udelay ( unsigned long usecs ) { + unsigned long start; + unsigned long elapsed; + unsigned long threshold; + + /* Delay until sufficient time has elapsed */ + start = rdtime_low(); + threshold = ( usecs * zicntr_mhz ); + do { + elapsed = ( rdtime_low() - start ); + } while ( elapsed < threshold ); +} + +/** + * Probe timer + * + * @ret rc Return status code + */ +static int zicntr_probe ( void ) { + unsigned int offset; + union { + uint64_t freq; + int64_t sfreq; + } u; + int rc; + + /* Check if time CSR can be read */ + if ( ! csr_can_read ( "time" ) ) { + DBGC ( colour, "ZICNTR cannot read TIME CSR\n" ); + return -ENOTSUP; + } + + /* Get timer frequency */ + if ( ( ( rc = fdt_path ( &sysfdt, "/cpus", &offset ) ) != 0 ) || + ( ( rc = fdt_u64 ( &sysfdt, offset, "timebase-frequency", + &u.freq ) ) != 0 ) ) { + DBGC ( colour, "ZICNTR could not determine frequency: %s\n", + strerror ( rc ) ); + return rc; + } + + /* Convert to MHz (without 64-bit division) */ + do { + zicntr_mhz++; + u.sfreq -= 1000000; + } while ( u.sfreq > 0 ); + + /* Calibrate currticks() scaling factor */ + zicntr_scale = 31; + zicntr_ticks = ( ( 1UL << zicntr_scale ) / + ( zicntr_mhz * ( 1000000 / TICKS_PER_SEC ) ) ); + while ( zicntr_ticks > ( TICKS_PER_SEC / ZICNTR_SCALED_HZ ) ) { + zicntr_scale--; + zicntr_ticks >>= 1; + } + DBGC ( colour, "ZICNTR at %ld MHz, %ld ticks per 2^%d increments\n", + zicntr_mhz, zicntr_ticks, zicntr_scale ); + if ( ! zicntr_ticks ) { + DBGC ( colour, "ZICNTR has zero ticks per 2^%d increments\n", + zicntr_scale ); + return -EIO; + } + + return 0; +} + +/** Zicntr timer */ +struct timer zicntr_timer __timer ( TIMER_PREFERRED ) = { + .name = "zicntr", + .probe = zicntr_probe, + .currticks = zicntr_currticks, + .udelay = zicntr_udelay, +}; diff --git a/src/arch/riscv/core/zkr.c b/src/arch/riscv/core/zkr.c new file mode 100644 index 000000000..b27a7f076 --- /dev/null +++ b/src/arch/riscv/core/zkr.c @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** @file + * + * Entropy source extension (Zkr) + * + */ + +#include <errno.h> +#include <ipxe/csr.h> +#include <ipxe/entropy.h> +#include <ipxe/drbg.h> + +struct entropy_source zkr_entropy __entropy_source ( ENTROPY_PREFERRED ); + +/** Seed CSR operational state */ +#define ZKR_SEED_OPST_MASK 0xc0000000UL +#define ZKR_SEED_OPST_ES16 0x80000000UL /**< 16 bits of entropy available */ + +/** Number of times to retry reading from seed CSR */ +#define ZKR_SEED_MAX_RETRY 1024 + +/** Colour for debug messages */ +#define colour &zkr_entropy + +/** + * Enable entropy gathering + * + * @ret rc Return status code + */ +static int zkr_entropy_enable ( void ) { + + /* Check if seed CSR is accessible in S-mode */ + if ( ! csr_can_write ( "seed", 0 ) ) { + DBGC ( colour, "ZKR cannot access seed CSR\n" ); + return -ENOTSUP; + } + + /* RISC-V ISA mandates that 128 bits of full entropy shall be + * obtained from 256 entropy bits read from the seed CSR. + * + * Each 16-bit sample therefore contains 8 bits of + * min-entropy. + */ + entropy_init ( &zkr_entropy, MIN_ENTROPY ( 8.0 ) ); + + return 0; +} + +/** + * Get noise sample + * + * @ret noise Noise sample + * @ret rc Return status code + */ +static int zkr_get_noise ( noise_sample_t *noise ) { + unsigned long seed; + unsigned int i; + + /* Read entropy from seed CSR */ + for ( i = 0 ; i < ZKR_SEED_MAX_RETRY ; i++ ) { + + /* Read seed CSR */ + __asm__ __volatile__ ( "csrrw %0, seed, zero" : + "=r" ( seed ) ); + + /* Check operationsl state */ + if ( ( seed & ZKR_SEED_OPST_MASK ) == ZKR_SEED_OPST_ES16 ) { + + /* Return entropy from both halves of the + * 16-bit entropy source value. + */ + *noise = ( seed ^ ( seed >> 8 ) ); + return 0; + } + } + + DBGC ( colour, "ZKR could not source entropy (seed %#08lx)\n", seed ); + return -EBUSY; +} + +/** Hardware entropy source */ +struct entropy_source zkr_entropy __entropy_source ( ENTROPY_PREFERRED ) = { + .name = "zkr", + .enable = zkr_entropy_enable, + .get_noise = zkr_get_noise, +}; diff --git a/src/arch/riscv/include/bits/bigint.h b/src/arch/riscv/include/bits/bigint.h new file mode 100644 index 000000000..7f87d9748 --- /dev/null +++ b/src/arch/riscv/include/bits/bigint.h @@ -0,0 +1,381 @@ +#ifndef _BITS_BIGINT_H +#define _BITS_BIGINT_H + +/** @file + * + * Big integer support + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> +#include <string.h> +#include <strings.h> + +/** Element of a big integer */ +typedef unsigned long bigint_element_t; + +/** + * Initialise big integer + * + * @v value0 Element 0 of big integer to initialise + * @v size Number of elements + * @v data Raw data + * @v len Length of raw data + */ +static inline __attribute__ (( always_inline )) void +bigint_init_raw ( unsigned long *value0, unsigned int size, + const void *data, size_t len ) { + size_t pad_len = ( sizeof ( bigint_t ( size ) ) - len ); + uint8_t *value_byte = ( ( void * ) value0 ); + const uint8_t *data_byte = ( data + len ); + + /* Copy raw data in reverse order, padding with zeros */ + while ( len-- ) + *(value_byte++) = *(--data_byte); + while ( pad_len-- ) + *(value_byte++) = 0; +} + +/** + * Add big integers + * + * @v addend0 Element 0 of big integer to add + * @v value0 Element 0 of big integer to be added to + * @v size Number of elements + * @ret carry Carry out + */ +static inline __attribute__ (( always_inline )) int +bigint_add_raw ( const unsigned long *addend0, unsigned long *value0, + unsigned int size ) { + bigint_t ( size ) __attribute__ (( may_alias )) *value = + ( ( void * ) value0 ); + unsigned long *valueN = ( value0 + size ); + unsigned long *discard_addend; + unsigned long *discard_value; + unsigned long discard_addend_i; + unsigned long discard_value_i; + unsigned long discard_temp; + unsigned long carry; + + __asm__ __volatile__ ( "\n1:\n\t" + /* Load addend[i] and value[i] */ + LOADN " %2, (%0)\n\t" + LOADN " %3, (%1)\n\t" + /* Add carry flag and addend */ + "add %3, %3, %5\n\t" + "sltu %4, %3, %5\n\t" + "add %3, %3, %2\n\t" + "sltu %5, %3, %2\n\t" + "or %5, %4, %5\n\t" + /* Store value[i] */ + STOREN " %3, (%1)\n\t" + /* Loop */ + "addi %0, %0, %8\n\t" + "addi %1, %1, %8\n\t" + "bne %1, %7, 1b\n\t" + : "=&r" ( discard_addend ), + "=&r" ( discard_value ), + "=&r" ( discard_addend_i ), + "=&r" ( discard_value_i ), + "=&r" ( discard_temp ), + "=&r" ( carry ), + "+m" ( *value ) + : "r" ( valueN ), + "i" ( sizeof ( unsigned long ) ), + "0" ( addend0 ), "1" ( value0 ), "5" ( 0 ) ); + return carry; +} + +/** + * Subtract big integers + * + * @v subtrahend0 Element 0 of big integer to subtract + * @v value0 Element 0 of big integer to be subtracted from + * @v size Number of elements + * @ret borrow Borrow out + */ +static inline __attribute__ (( always_inline )) int +bigint_subtract_raw ( const unsigned long *subtrahend0, unsigned long *value0, + unsigned int size ) { + bigint_t ( size ) __attribute__ (( may_alias )) *value = + ( ( void * ) value0 ); + unsigned long *valueN = ( value0 + size ); + unsigned long *discard_subtrahend; + unsigned long *discard_value; + unsigned long discard_subtrahend_i; + unsigned long discard_value_i; + unsigned long discard_temp; + unsigned long borrow; + + __asm__ __volatile__ ( "\n1:\n\t" + /* Load subtrahend[i] and value[i] */ + LOADN " %2, (%0)\n\t" + LOADN " %3, (%1)\n\t" + /* Subtract carry flag and subtrahend */ + "sltu %4, %3, %5\n\t" + "sub %3, %3, %5\n\t" + "sltu %5, %3, %2\n\t" + "sub %3, %3, %2\n\t" + "or %5, %5, %4\n\t" + /* Store value[i] */ + STOREN " %3, (%1)\n\t" + /* Loop */ + "addi %0, %0, %8\n\t" + "addi %1, %1, %8\n\t" + "bne %1, %7, 1b\n\t" + : "=&r" ( discard_subtrahend ), + "=&r" ( discard_value ), + "=&r" ( discard_subtrahend_i ), + "=&r" ( discard_value_i ), + "=&r" ( discard_temp ), + "=&r" ( borrow ), + "+m" ( *value ) + : "r" ( valueN ), + "i" ( sizeof ( unsigned long ) ), + "0" ( subtrahend0 ), "1" ( value0 ), + "5" ( 0 ) ); + return borrow; +} + +/** + * Shift big integer left + * + * @v value0 Element 0 of big integer + * @v size Number of elements + * @ret out Bit shifted out + */ +static inline __attribute__ (( always_inline )) int +bigint_shl_raw ( unsigned long *value0, unsigned int size ) { + bigint_t ( size ) __attribute__ (( may_alias )) *value = + ( ( void * ) value0 ); + unsigned long *valueN = ( value0 + size ); + unsigned long *discard_value; + unsigned long discard_value_i; + unsigned long discard_temp; + unsigned long carry; + + __asm__ __volatile__ ( "\n1:\n\t" + /* Load value[i] */ + LOADN " %1, (%0)\n\t" + /* Shift left */ + "slli %2, %1, 1\n\t" + "or %2, %2, %3\n\t" + "srli %3, %1, %7\n\t" + /* Store value[i] */ + STOREN " %2, (%0)\n\t" + /* Loop */ + "addi %0, %0, %6\n\t" + "bne %0, %5, 1b\n\t" + : "=&r" ( discard_value ), + "=&r" ( discard_value_i ), + "=&r" ( discard_temp ), + "=&r" ( carry ), + "+m" ( *value ) + : "r" ( valueN ), + "i" ( sizeof ( unsigned long ) ), + "i" ( ( 8 * sizeof ( unsigned long ) - 1 ) ), + "0" ( value0 ), "3" ( 0 ) ); + return carry; +} + +/** + * Shift big integer right + * + * @v value0 Element 0 of big integer + * @v size Number of elements + * @ret out Bit shifted out + */ +static inline __attribute__ (( always_inline )) int +bigint_shr_raw ( unsigned long *value0, unsigned int size ) { + bigint_t ( size ) __attribute__ (( may_alias )) *value = + ( ( void * ) value0 ); + unsigned long *valueN = ( value0 + size ); + unsigned long *discard_value; + unsigned long discard_value_i; + unsigned long discard_temp; + unsigned long carry; + + __asm__ __volatile__ ( "\n1:\n\t" + /* Load value[i] */ + LOADN " %1, %6(%0)\n\t" + /* Shift right */ + "srli %2, %1, 1\n\t" + "or %2, %2, %3\n\t" + "slli %3, %1, %7\n\t" + /* Store value[i] */ + STOREN " %2, %6(%0)\n\t" + /* Loop */ + "addi %0, %0, %6\n\t" + "bne %0, %5, 1b\n\t" + : "=&r" ( discard_value ), + "=&r" ( discard_value_i ), + "=&r" ( discard_temp ), + "=&r" ( carry ), + "+m" ( *value ) + : "r" ( value0 ), + "i" ( -( sizeof ( unsigned long ) ) ), + "i" ( ( 8 * sizeof ( unsigned long ) - 1 ) ), + "0" ( valueN ), "3" ( 0 ) ); + return ( !! carry ); +} + +/** + * Test if big integer is equal to zero + * + * @v value0 Element 0 of big integer + * @v size Number of elements + * @ret is_zero Big integer is equal to zero + */ +static inline __attribute__ (( always_inline, pure )) int +bigint_is_zero_raw ( const unsigned long *value0, unsigned int size ) { + const unsigned long *value = value0; + unsigned long value_i; + + do { + value_i = *(value++); + if ( value_i ) + break; + } while ( --size ); + + return ( value_i == 0 ); +} + +/** + * Compare big integers + * + * @v value0 Element 0 of big integer + * @v reference0 Element 0 of reference big integer + * @v size Number of elements + * @ret geq Big integer is greater than or equal to the reference + */ +static inline __attribute__ (( always_inline, pure )) int +bigint_is_geq_raw ( const unsigned long *value0, + const unsigned long *reference0, unsigned int size ) { + const unsigned long *value = ( value0 + size ); + const unsigned long *reference = ( reference0 + size ); + unsigned long value_i; + unsigned long reference_i; + + do { + value_i = *(--value); + reference_i = *(--reference); + if ( value_i != reference_i ) + break; + } while ( --size ); + + return ( value_i >= reference_i ); +} + +/** + * Find highest bit set in big integer + * + * @v value0 Element 0 of big integer + * @v size Number of elements + * @ret max_bit Highest bit set + 1 (or 0 if no bits set) + */ +static inline __attribute__ (( always_inline )) int +bigint_max_set_bit_raw ( const unsigned long *value0, unsigned int size ) { + const unsigned long *value = ( value0 + size ); + int max_bit = ( 8 * sizeof ( bigint_t ( size ) ) ); + unsigned long value_i; + + do { + value_i = *(--value); + max_bit -= ( ( 8 * sizeof ( *value0 ) ) - fls ( value_i ) ); + if ( value_i ) + break; + } while ( --size ); + + return max_bit; +} + +/** + * Grow big integer + * + * @v source0 Element 0 of source big integer + * @v source_size Number of elements in source big integer + * @v dest0 Element 0 of destination big integer + * @v dest_size Number of elements in destination big integer + */ +static inline __attribute__ (( always_inline )) void +bigint_grow_raw ( const unsigned long *source0, unsigned int source_size, + unsigned long *dest0, unsigned int dest_size ) { + unsigned int pad_size = ( dest_size - source_size ); + + memcpy ( dest0, source0, sizeof ( bigint_t ( source_size ) ) ); + memset ( ( dest0 + source_size ), 0, sizeof ( bigint_t ( pad_size ) ) ); +} + +/** + * Shrink big integer + * + * @v source0 Element 0 of source big integer + * @v source_size Number of elements in source big integer + * @v dest0 Element 0 of destination big integer + * @v dest_size Number of elements in destination big integer + */ +static inline __attribute__ (( always_inline )) void +bigint_shrink_raw ( const unsigned long *source0, + unsigned int source_size __unused, + unsigned long *dest0, unsigned int dest_size ) { + + memcpy ( dest0, source0, sizeof ( bigint_t ( dest_size ) ) ); +} + +/** + * Finalise big integer + * + * @v value0 Element 0 of big integer to finalise + * @v size Number of elements + * @v out Output buffer + * @v len Length of output buffer + */ +static inline __attribute__ (( always_inline )) void +bigint_done_raw ( const unsigned long *value0, unsigned int size __unused, + void *out, size_t len ) { + const uint8_t *value_byte = ( ( const void * ) value0 ); + uint8_t *out_byte = ( out + len ); + + /* Copy raw data in reverse order */ + while ( len-- ) + *(--out_byte) = *(value_byte++); +} + +/** + * Multiply big integer elements + * + * @v multiplicand Multiplicand element + * @v multiplier Multiplier element + * @v result Result element + * @v carry Carry element + */ +static inline __attribute__ (( always_inline )) void +bigint_multiply_one ( const unsigned long multiplicand, + const unsigned long multiplier, + unsigned long *result, unsigned long *carry ) { + unsigned long discard_low; + unsigned long discard_high; + unsigned long discard_carry; + + __asm__ __volatile__ ( /* Perform multiplication */ + "mulhu %1, %5, %6\n\t" + "mul %0, %5, %6\n\t" + /* Accumulate low half */ + "add %3, %3, %0\n\t" + "sltu %2, %3, %0\n\t" + "add %1, %1, %2\n\t" + /* Accumulate carry (cannot overflow) */ + "add %3, %3, %4\n\t" + "sltu %2, %3, %4\n\t" + "add %4, %1, %2\n\t" + : "=r" ( discard_low ), + "=&r" ( discard_high ), + "=r" ( discard_carry ), + "+r" ( *result ), + "+r" ( *carry ) + : "r" ( multiplicand ), + "r" ( multiplier ) ); +} + +#endif /* _BITS_BIGINT_H */ diff --git a/src/arch/riscv/include/bits/bitops.h b/src/arch/riscv/include/bits/bitops.h new file mode 100644 index 000000000..2019db99a --- /dev/null +++ b/src/arch/riscv/include/bits/bitops.h @@ -0,0 +1,82 @@ +#ifndef _BITS_BITOPS_H +#define _BITS_BITOPS_H + +/** @file + * + * RISC-V bit operations + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> + +/** + * Test and set bit atomically + * + * @v bit Bit to set + * @v bits Bit field + * @ret old Old value of bit (zero or non-zero) + */ +static inline __attribute__ (( always_inline )) int +test_and_set_bit ( unsigned int bit, volatile void *bits ) { + unsigned int index = ( bit / 32 ); + unsigned int offset = ( bit % 32 ); + volatile uint32_t *word = ( ( ( volatile uint32_t * ) bits ) + index ); + uint32_t mask = ( 1U << offset ); + uint32_t old; + + __asm__ __volatile__ ( "amoor.w %0, %2, %1" + : "=r" ( old ), "+A" ( *word ) + : "r" ( mask ) ); + + return ( !! ( old & mask ) ); +} + +/** + * Test and clear bit atomically + * + * @v bit Bit to set + * @v bits Bit field + * @ret old Old value of bit (zero or non-zero) + */ +static inline __attribute__ (( always_inline )) int +test_and_clear_bit ( unsigned int bit, volatile void *bits ) { + unsigned int index = ( bit / 32 ); + unsigned int offset = ( bit % 32 ); + volatile uint32_t *word = ( ( ( volatile uint32_t * ) bits ) + index ); + uint32_t mask = ( 1U << offset ); + uint32_t old; + + __asm__ __volatile__ ( "amoand.w %0, %2, %1" + : "=r" ( old ), "+A" ( *word ) + : "r" ( ~mask ) ); + + return ( !! ( old & mask ) ); +} + +/** + * Set bit atomically + * + * @v bit Bit to set + * @v bits Bit field + */ +static inline __attribute__ (( always_inline )) void +set_bit ( unsigned int bit, volatile void *bits ) { + + test_and_set_bit ( bit, bits ); +} + +/** + * Clear bit atomically + * + * @v bit Bit to set + * @v bits Bit field + */ +static inline __attribute__ (( always_inline )) void +clear_bit ( unsigned int bit, volatile void *bits ) { + + test_and_clear_bit ( bit, bits ); +} + +#endif /* _BITS_BITOPS_H */ diff --git a/src/arch/riscv/include/bits/byteswap.h b/src/arch/riscv/include/bits/byteswap.h new file mode 100644 index 000000000..56d03f64e --- /dev/null +++ b/src/arch/riscv/include/bits/byteswap.h @@ -0,0 +1,48 @@ +#ifndef _BITS_BYTESWAP_H +#define _BITS_BYTESWAP_H + +/** @file + * + * Byte-order swapping functions + * + */ + +#include <stdint.h> + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +extern __asmcall uint64_t riscv_swap_word ( uint64_t x ); +extern __asmcall unsigned long riscv_swap_half ( unsigned long x ); +extern __asmcall unsigned long riscv_swap_byte ( unsigned long x ); + +static inline __attribute__ (( always_inline, const )) uint16_t +__bswap_variable_16 ( uint16_t x ) { + return riscv_swap_byte ( x ); +} + +static inline __attribute__ (( always_inline )) void +__bswap_16s ( uint16_t *x ) { + *x = riscv_swap_byte ( *x ); +} + +static inline __attribute__ (( always_inline, const )) uint32_t +__bswap_variable_32 ( uint32_t x ) { + return riscv_swap_half ( x ); +} + +static inline __attribute__ (( always_inline )) void +__bswap_32s ( uint32_t *x ) { + *x = riscv_swap_half ( *x ); +} + +static inline __attribute__ (( always_inline, const )) uint64_t +__bswap_variable_64 ( uint64_t x ) { + return riscv_swap_word ( x ); +} + +static inline __attribute__ (( always_inline )) void +__bswap_64s ( uint64_t *x ) { + *x = riscv_swap_word ( *x ); +} + +#endif /* _BITS_BYTESWAP_H */ diff --git a/src/arch/riscv/include/bits/compiler.h b/src/arch/riscv/include/bits/compiler.h new file mode 100644 index 000000000..624a16108 --- /dev/null +++ b/src/arch/riscv/include/bits/compiler.h @@ -0,0 +1,40 @@ +#ifndef _BITS_COMPILER_H +#define _BITS_COMPILER_H + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** Dummy relocation type */ +#define RELOC_TYPE_NONE R_RISCV_NONE + +/* Determine load/store instructions for natural bit width */ +#if __riscv_xlen == 128 +#define NATURAL_SUFFIX q +#elif __riscv_xlen == 64 +#define NATURAL_SUFFIX d +#elif __riscv_xlen == 32 +#define NATURAL_SUFFIX w +#else +#error "Unsupported bit width" +#endif +#ifdef ASSEMBLY +#define LOADN _C2 ( L, NATURAL_SUFFIX ) +#define STOREN _C2 ( S, NATURAL_SUFFIX ) +#else +#define LOADN "L" _S2 ( NATURAL_SUFFIX ) +#define STOREN "S" _S2 ( NATURAL_SUFFIX ) +#endif + +#ifndef ASSEMBLY + +/** Unprefixed constant operand modifier */ +#define ASM_NO_PREFIX "" + +/** Declare a function with standard calling conventions */ +#define __asmcall + +/** Declare a function with libgcc implicit linkage */ +#define __libgcc + +#endif /* ASSEMBLY */ + +#endif /* _BITS_COMPILER_H */ diff --git a/src/arch/riscv/include/bits/dma.h b/src/arch/riscv/include/bits/dma.h new file mode 100644 index 000000000..f7decd14c --- /dev/null +++ b/src/arch/riscv/include/bits/dma.h @@ -0,0 +1,14 @@ +#ifndef _BITS_DMA_H +#define _BITS_DMA_H + +/** @file + * + * RISCV-specific DMA API implementations + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <ipxe/riscv_dma.h> + +#endif /* _BITS_DMA_H */ diff --git a/src/arch/riscv/include/bits/endian.h b/src/arch/riscv/include/bits/endian.h new file mode 100644 index 000000000..85718cfdd --- /dev/null +++ b/src/arch/riscv/include/bits/endian.h @@ -0,0 +1,8 @@ +#ifndef _BITS_ENDIAN_H +#define _BITS_ENDIAN_H + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#define __BYTE_ORDER __LITTLE_ENDIAN + +#endif /* _BITS_ENDIAN_H */ diff --git a/src/arch/riscv/include/bits/errfile.h b/src/arch/riscv/include/bits/errfile.h new file mode 100644 index 000000000..bdd2927a4 --- /dev/null +++ b/src/arch/riscv/include/bits/errfile.h @@ -0,0 +1,24 @@ +#ifndef _BITS_ERRFILE_H +#define _BITS_ERRFILE_H + +/** @file + * + * RISC-V error file identifiers + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** + * @addtogroup errfile Error file identifiers + * @{ + */ + +#define ERRFILE_sbi_reboot ( ERRFILE_ARCH | ERRFILE_CORE | 0x00000000 ) +#define ERRFILE_hart ( ERRFILE_ARCH | ERRFILE_CORE | 0x00010000 ) +#define ERRFILE_zicntr ( ERRFILE_ARCH | ERRFILE_CORE | 0x00020000 ) +#define ERRFILE_zkr ( ERRFILE_ARCH | ERRFILE_CORE | 0x00030000 ) + +/** @} */ + +#endif /* _BITS_ERRFILE_H */ diff --git a/src/arch/riscv/include/bits/io.h b/src/arch/riscv/include/bits/io.h new file mode 100644 index 000000000..4296e318a --- /dev/null +++ b/src/arch/riscv/include/bits/io.h @@ -0,0 +1,17 @@ +#ifndef _BITS_IO_H +#define _BITS_IO_H + +/** @file + * + * RISCV-specific I/O API implementations + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** Page shift */ +#define PAGE_SHIFT 12 + +#include <ipxe/riscv_io.h> + +#endif /* _BITS_IO_H */ diff --git a/src/arch/loong64/include/bits/iomap.h b/src/arch/riscv/include/bits/iomap.h index 041171d22..fd8e37825 100644 --- a/src/arch/loong64/include/bits/iomap.h +++ b/src/arch/riscv/include/bits/iomap.h @@ -3,10 +3,12 @@ /** @file * - * LoongArch64-specific I/O mapping API implementations + * RISCV-specific I/O mapping API implementations * */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +#include <ipxe/svpage.h> + #endif /* _BITS_IOMAP_H */ diff --git a/src/arch/riscv/include/bits/lkrn.h b/src/arch/riscv/include/bits/lkrn.h new file mode 100644 index 000000000..d26108647 --- /dev/null +++ b/src/arch/riscv/include/bits/lkrn.h @@ -0,0 +1,34 @@ +#ifndef _BITS_LKRN_H +#define _BITS_LKRN_H + +/** @file + * + * Linux kernel image invocation + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <ipxe/hart.h> + +/** Header magic value */ +#define LKRN_MAGIC_ARCH LKRN_MAGIC_RISCV + +/** + * Jump to kernel entry point + * + * @v entry Kernel entry point + * @v fdt Device tree + */ +static inline __attribute__ (( noreturn )) void +lkrn_jump ( physaddr_t entry, physaddr_t fdt ) { + register unsigned long a0 asm ( "a0" ) = boot_hart; + register unsigned long a1 asm ( "a1" ) = fdt; + + __asm__ __volatile__ ( "call disable_paging\n\t" + "jr %2\n\t" + : : "r" ( a0 ), "r" ( a1 ), "r" ( entry ) ); + __builtin_unreachable(); +} + +#endif /* _BITS_LKRN_H */ diff --git a/src/arch/riscv/include/bits/nap.h b/src/arch/riscv/include/bits/nap.h new file mode 100644 index 000000000..331399f46 --- /dev/null +++ b/src/arch/riscv/include/bits/nap.h @@ -0,0 +1,20 @@ +#ifndef _BITS_NAP_H +#define _BITS_NAP_H + +/** @file + * + * RISCV-specific CPU sleeping API implementations + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** + * Sleep until next CPU interrupt + * + */ +static inline __attribute__ (( always_inline )) void cpu_halt ( void ) { + __asm__ __volatile__ ( "wfi" ); +} + +#endif /* _BITS_NAP_H */ diff --git a/src/arch/riscv/include/bits/profile.h b/src/arch/riscv/include/bits/profile.h new file mode 100644 index 000000000..e9e003dab --- /dev/null +++ b/src/arch/riscv/include/bits/profile.h @@ -0,0 +1,28 @@ +#ifndef _BITS_PROFILE_H +#define _BITS_PROFILE_H + +/** @file + * + * Profiling + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> + +/** + * Get profiling timestamp + * + * @ret timestamp Timestamp + */ +static inline __attribute__ (( always_inline )) unsigned long +profile_timestamp ( void ) { + unsigned long cycles; + + /* Read timestamp counter */ + __asm__ __volatile__ ( "rdcycle %0" : "=r" ( cycles ) ); + return cycles; +} + +#endif /* _BITS_PROFILE_H */ diff --git a/src/arch/arm/include/bits/reboot.h b/src/arch/riscv/include/bits/reboot.h index 88c50250c..01272483b 100644 --- a/src/arch/arm/include/bits/reboot.h +++ b/src/arch/riscv/include/bits/reboot.h @@ -3,10 +3,12 @@ /** @file * - * ARM-specific reboot API implementations + * RISCV-specific reboot API implementations * */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +#include <ipxe/sbi_reboot.h> + #endif /* _BITS_REBOOT_H */ diff --git a/src/arch/riscv/include/bits/setjmp.h b/src/arch/riscv/include/bits/setjmp.h new file mode 100644 index 000000000..5186fadaf --- /dev/null +++ b/src/arch/riscv/include/bits/setjmp.h @@ -0,0 +1,16 @@ +#ifndef _BITS_SETJMP_H +#define _BITS_SETJMP_H + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** A jump buffer */ +typedef struct { + /** Return address (ra) */ + unsigned long ra; + /** Stack pointer (sp) */ + unsigned long sp; + /** Callee-saved registers (s0-s11) */ + unsigned long s[12]; +} jmp_buf[1]; + +#endif /* _BITS_SETJMP_H */ diff --git a/src/arch/riscv/include/bits/stdint.h b/src/arch/riscv/include/bits/stdint.h new file mode 100644 index 000000000..fe1f9946a --- /dev/null +++ b/src/arch/riscv/include/bits/stdint.h @@ -0,0 +1,23 @@ +#ifndef _BITS_STDINT_H +#define _BITS_STDINT_H + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +typedef __SIZE_TYPE__ size_t; +typedef signed long ssize_t; +typedef signed long off_t; + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +typedef signed char int8_t; +typedef signed short int16_t; +typedef signed int int32_t; +typedef signed long long int64_t; + +typedef unsigned long physaddr_t; +typedef unsigned long intptr_t; + +#endif /* _BITS_STDINT_H */ diff --git a/src/arch/riscv/include/bits/string.h b/src/arch/riscv/include/bits/string.h new file mode 100644 index 000000000..87834d91a --- /dev/null +++ b/src/arch/riscv/include/bits/string.h @@ -0,0 +1,82 @@ +#ifndef _BITS_STRING_H +#define _BITS_STRING_H + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** @file + * + * String functions + * + */ + +extern void riscv_bzero ( void *dest, size_t len ); +extern void riscv_memset ( void *dest, size_t len, int character ); +extern void riscv_memcpy ( void *dest, const void *src, size_t len ); +extern void riscv_memmove ( void *dest, const void *src, size_t len ); + +/** + * Fill memory region + * + * @v dest Destination region + * @v character Fill character + * @v len Length + * @ret dest Destination region + */ +static inline __attribute__ (( always_inline )) void * +memset ( void *dest, int character, size_t len ) { + + /* For zeroing larger or non-constant lengths, use the + * optimised variable-length zeroing code. + */ + if ( __builtin_constant_p ( character ) && ( character == 0 ) ) { + riscv_bzero ( dest, len ); + return dest; + } + + /* Not necessarily zeroing: use basic variable-length code */ + riscv_memset ( dest, len, character ); + return dest; +} + +/** + * Copy memory region + * + * @v dest Destination region + * @v src Source region + * @v len Length + * @ret dest Destination region + */ +static inline __attribute__ (( always_inline )) void * +memcpy ( void *dest, const void *src, size_t len ) { + + /* Otherwise, use variable-length code */ + riscv_memcpy ( dest, src, len ); + return dest; +} + +/** + * Copy (possibly overlapping) memory region + * + * @v dest Destination region + * @v src Source region + * @v len Length + * @ret dest Destination region + */ +static inline __attribute__ (( always_inline )) void * +memmove ( void *dest, const void *src, size_t len ) { + ssize_t offset = ( dest - src ); + + /* If direction of copy is known to be forwards at build time, + * then use variable-length memcpy(). + */ + if ( __builtin_constant_p ( offset ) && ( offset <= 0 ) ) { + riscv_memcpy ( dest, src, len ); + return dest; + } + + /* Otherwise, use ambidirectional copy */ + riscv_memmove ( dest, src, len ); + return dest; +} + +#endif /* _BITS_STRING_H */ diff --git a/src/arch/riscv/include/bits/strings.h b/src/arch/riscv/include/bits/strings.h new file mode 100644 index 000000000..dd6d458b2 --- /dev/null +++ b/src/arch/riscv/include/bits/strings.h @@ -0,0 +1,91 @@ +#ifndef _BITS_STRINGS_H +#define _BITS_STRINGS_H + +/** @file + * + * String functions + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +extern __asmcall unsigned long riscv_ffs ( unsigned long value ); +extern __asmcall unsigned long riscv_fls ( unsigned long value ); + +/** + * Find first (i.e. least significant) set bit + * + * @v value Value + * @ret lsb Least significant bit set in value (LSB=1), or zero + */ +static inline __attribute__ (( always_inline )) int __ffsl ( long value ) { + + return riscv_ffs ( value ); +} + +/** + * Find first (i.e. least significant) set bit + * + * @v value Value + * @ret lsb Least significant bit set in value (LSB=1), or zero + */ +static inline __attribute__ (( always_inline )) int __ffsll ( long long value ){ + unsigned long low = value; + unsigned long high; + + /* Check machine word size */ + if ( sizeof ( value ) > sizeof ( low ) ) { + /* 32-bit */ + high = ( value >> 32 ); + if ( low ) { + return ( __ffsl ( low ) ); + } else if ( high ) { + return ( 32 + __ffsl ( high ) ); + } else { + return 0; + } + } else { + /* 64-bit */ + return ( __ffsl ( low ) ); + } +} + +/** + * Find last (i.e. most significant) set bit + * + * @v value Value + * @ret msb Most significant bit set in value (LSB=1), or zero + */ +static inline __attribute__ (( always_inline )) int __flsl ( long value ) { + + return riscv_fls ( value ); +} + +/** + * Find last (i.e. most significant) set bit + * + * @v value Value + * @ret msb Most significant bit set in value (LSB=1), or zero + */ +static inline __attribute__ (( always_inline )) int __flsll ( long long value ){ + unsigned long low = value; + unsigned long high; + + /* Check machine word size */ + if ( sizeof ( value ) > sizeof ( low ) ) { + /* 32-bit */ + high = ( value >> 32 ); + if ( high ) { + return ( 32 + __flsl ( high ) ); + } else if ( low ) { + return ( __flsl ( low ) ); + } else { + return 0; + } + } else { + /* 64-bit */ + return ( __flsl ( low ) ); + } +} + +#endif /* _BITS_STRINGS_H */ diff --git a/src/arch/riscv/include/bits/tcpip.h b/src/arch/riscv/include/bits/tcpip.h new file mode 100644 index 000000000..0ac55b1a0 --- /dev/null +++ b/src/arch/riscv/include/bits/tcpip.h @@ -0,0 +1,15 @@ +#ifndef _BITS_TCPIP_H +#define _BITS_TCPIP_H + +/** @file + * + * Transport-network layer interface + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +extern uint16_t tcpip_continue_chksum ( uint16_t partial, const void *data, + size_t len ); + +#endif /* _BITS_TCPIP_H */ diff --git a/src/arch/riscv/include/bits/virt_offset.h b/src/arch/riscv/include/bits/virt_offset.h new file mode 100644 index 000000000..83ac17551 --- /dev/null +++ b/src/arch/riscv/include/bits/virt_offset.h @@ -0,0 +1,33 @@ +#ifndef _BITS_VIRT_OFFSET_H +#define _BITS_VIRT_OFFSET_H + +/** @file + * + * RISCV-specific virtual address offset + * + * We use the thread pointer register (tp) to hold the virtual address + * offset, so that virtual-to-physical address translations work as + * expected even while we are executing directly from read-only memory + * (and so cannot store a value in a global virt_offset variable). + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** + * Read virtual address offset held in thread pointer register + * + * @ret virt_offset Virtual address offset + */ +static inline __attribute__ (( const, always_inline )) unsigned long +tp_virt_offset ( void ) { + register unsigned long tp asm ( "tp" ); + + __asm__ ( "" : "=r" ( tp ) ); + return tp; +} + +/** Always read thread pointer register to get virtual address offset */ +#define virt_offset tp_virt_offset() + +#endif /* _BITS_VIRT_OFFSET_H */ diff --git a/src/arch/riscv/include/ipxe/csr.h b/src/arch/riscv/include/ipxe/csr.h new file mode 100644 index 000000000..c14974472 --- /dev/null +++ b/src/arch/riscv/include/ipxe/csr.h @@ -0,0 +1,75 @@ +#ifndef _IPXE_CSR_H +#define _IPXE_CSR_H + +/** @file + * + * Control and status registers (CSRs) + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** + * Check if CSR can be read + * + * @v name CSR name + * @v allowed CSR can be read + */ +#define csr_can_read( name ) ( { \ + unsigned long stvec_orig; \ + unsigned long stvec_temp; \ + unsigned long csr; \ + int allowed = 0; \ + \ + __asm__ __volatile__ ( /* Set temporary trap vector */ \ + "la %3, 1f\n\t" \ + "csrrw %2, stvec, %3\n\t" \ + /* Try reading CSR */ \ + "csrr %1, " name "\n\t" \ + /* Mark as allowed if not trapped */ \ + "addi %0, %0, 1\n\t" \ + /* Temporary trap vector */ \ + ".balign 4\n\t" \ + "\n1:\n\t" \ + /* Restore original trap vector */ \ + "csrw stvec, %2\n\t" \ + : "+r" ( allowed ), \ + "=r" ( csr ), \ + "=r" ( stvec_orig ), \ + "=r" ( stvec_temp ) ); \ + allowed; \ + } ) + +/** + * Check if CSR can be written + * + * @v name CSR name + * @v value Value to write + * @v allowed CSR can be written + */ +#define csr_can_write( name, value ) ( { \ + unsigned long stvec_orig; \ + unsigned long stvec_temp; \ + unsigned long csr = (value); \ + int allowed = 0; \ + \ + __asm__ __volatile__ ( /* Set temporary trap vector */ \ + "la %3, 1f\n\t" \ + "csrrw %2, stvec, %3\n\t" \ + /* Try writing CSR */ \ + "csrrw %1, " name ", %1\n\t" \ + /* Mark as allowed if not trapped */ \ + "addi %0, %0, 1\n\t" \ + /* Temporary trap vector */ \ + ".balign 4\n\t" \ + "\n1:\n\t" \ + /* Restore original trap vector */ \ + "csrw stvec, %2\n\t" \ + : "+r" ( allowed ), \ + "+r" ( csr ), \ + "=r" ( stvec_orig ), \ + "=r" ( stvec_temp ) ); \ + allowed; \ + } ) + +#endif /* _IPXE_CSR_H */ diff --git a/src/arch/riscv/include/ipxe/errno/sbi.h b/src/arch/riscv/include/ipxe/errno/sbi.h new file mode 100644 index 000000000..2428183d4 --- /dev/null +++ b/src/arch/riscv/include/ipxe/errno/sbi.h @@ -0,0 +1,19 @@ +#ifndef _IPXE_ERRNO_SBI_H +#define _IPXE_ERRNO_SBI_H + +/** + * @file + * + * RISC-V SBI platform error codes + * + * We never need to return SBI error codes ourselves, so we + * arbitrarily choose to use the Linux error codes as platform error + * codes. + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <ipxe/errno/linux.h> + +#endif /* _IPXE_ERRNO_SBI_H */ diff --git a/src/arch/riscv/include/ipxe/hart.h b/src/arch/riscv/include/ipxe/hart.h new file mode 100644 index 000000000..c201b6c77 --- /dev/null +++ b/src/arch/riscv/include/ipxe/hart.h @@ -0,0 +1,16 @@ +#ifndef _IPXE_HART_H +#define _IPXE_HART_H + +/** @file + * + * Hardware threads (harts) + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +extern unsigned long boot_hart; + +extern int hart_supported ( const char *ext ); + +#endif /* _IPXE_HART_H */ diff --git a/src/arch/riscv/include/ipxe/riscv_dma.h b/src/arch/riscv/include/ipxe/riscv_dma.h new file mode 100644 index 000000000..d35904d88 --- /dev/null +++ b/src/arch/riscv/include/ipxe/riscv_dma.h @@ -0,0 +1,45 @@ +#ifndef _IPXE_RISCV_DMA_H +#define _IPXE_RISCV_DMA_H + +/** @file + * + * iPXE DMA API for RISC-V + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#ifdef DMAAPI_RISCV +#define DMAAPI_PREFIX_riscv +#else +#define DMAAPI_PREFIX_riscv __riscv_ +#endif + +/** + * Set addressable space mask + * + * @v dma DMA device + * @v mask Addressable space mask + */ +static inline __always_inline void +DMAAPI_INLINE ( riscv, dma_set_mask ) ( struct dma_device *dma __unused, + physaddr_t mask __unused ) { + + /* Nothing to do */ +} + +/** + * Get DMA address from virtual address + * + * @v map DMA mapping + * @v addr Address within the mapped region + * @ret addr Device-side DMA address + */ +static inline __always_inline physaddr_t +DMAAPI_INLINE ( riscv, dma ) ( struct dma_mapping *map __unused, void *addr ) { + + /* Use physical address as device address */ + return virt_to_phys ( addr ); +} + +#endif /* _IPXE_RISCV_DMA_H */ diff --git a/src/arch/riscv/include/ipxe/riscv_io.h b/src/arch/riscv/include/ipxe/riscv_io.h new file mode 100644 index 000000000..539dbd7ed --- /dev/null +++ b/src/arch/riscv/include/ipxe/riscv_io.h @@ -0,0 +1,141 @@ +#ifndef _IPXE_RISCV_IO_H +#define _IPXE_RISCV_IO_H + +/** @file + * + * iPXE I/O API for RISC-V + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#ifdef IOAPI_RISCV +#define IOAPI_PREFIX_riscv +#else +#define IOAPI_PREFIX_riscv __riscv_ +#endif + +#include <ipxe/dummy_pio.h> + +/* + * Memory space mappings + * + */ + +/* + * Physical<->Bus address mappings + * + */ + +static inline __always_inline unsigned long +IOAPI_INLINE ( riscv, phys_to_bus ) ( unsigned long phys_addr ) { + return phys_addr; +} + +static inline __always_inline unsigned long +IOAPI_INLINE ( riscv, bus_to_phys ) ( unsigned long bus_addr ) { + return bus_addr; +} + +/* + * MMIO reads and writes + * + */ + +/* Single-register read */ +#define RISCV_READX( _suffix, _type, _insn_suffix ) \ +static inline __always_inline _type \ +IOAPI_INLINE ( riscv, read ## _suffix ) ( volatile _type *io_addr ) { \ + unsigned long data; \ + __asm__ __volatile__ ( "fence io, io\n\t" \ + "l" _insn_suffix " %0, %1\n\t" \ + : "=r" ( data ) : "m" ( *io_addr ) ); \ + return data; \ +} + +/* Single-register write */ +#define RISCV_WRITEX( _suffix, _type, _insn_suffix) \ +static inline __always_inline void \ +IOAPI_INLINE ( riscv, write ## _suffix ) ( _type data, \ + volatile _type *io_addr ) { \ + __asm__ __volatile__ ( "fence io, io\n\t" \ + "s" _insn_suffix " %0, %1\n\t" \ + : : "r" ( data ), "m" ( *io_addr ) ); \ +} + +/* Double-register hopefully-fused read */ +#define RISCV_READX_FUSED( _suffix, _type, _insn_suffix ) \ +static inline __always_inline _type \ +IOAPI_INLINE ( riscv, read ## _suffix ) ( volatile _type *io_addr ) { \ + union { \ + unsigned long half[2]; \ + _type data; \ + } u; \ + __asm__ __volatile__ ( "fence io, io\n\t" \ + "l" _insn_suffix " %0, 0(%2)\n\t" \ + "l" _insn_suffix " %1, %3(%2)\n\t" \ + : "=&r" ( u.half[0] ), \ + "=&r" ( u.half[1] ) \ + : "r" ( io_addr ), \ + "i" ( sizeof ( u.half[0] ) ) ); \ + return u.data; \ +} + +/* Double-register hopefully-fused write */ +#define RISCV_WRITEX_FUSED( _suffix, _type, _insn_suffix ) \ +static inline __always_inline void \ +IOAPI_INLINE ( riscv, write ## _suffix ) ( _type data, \ + volatile _type *io_addr ) { \ + union { \ + unsigned long half[2]; \ + _type data; \ + } u = { .data = data }; \ + __asm__ __volatile__ ( "fence io, io\n\t" \ + "s" _insn_suffix " %0, 0(%2)\n\t" \ + "s" _insn_suffix " %1, %3(%2)\n\t" : \ + : "r" ( u.half[0] ), \ + "r" ( u.half[1] ), \ + "r" ( io_addr ), \ + "i" ( sizeof ( u.half[0] ) ) ); \ +} + +RISCV_READX ( b, uint8_t, "bu" ); +RISCV_WRITEX ( b, uint8_t, "b" ); + +RISCV_READX ( w, uint16_t, "hu" ); +RISCV_WRITEX ( w, uint16_t, "h" ); + +#if __riscv_xlen > 32 + RISCV_READX ( l, uint32_t, "wu" ); + RISCV_WRITEX ( l, uint32_t, "w" ); +#else + RISCV_READX ( l, uint32_t, "w" ); + RISCV_WRITEX ( l, uint32_t, "w" ); +#endif + +#if __riscv_xlen >= 64 + #if __riscv_xlen > 64 + RISCV_READX ( q, uint64_t, "du" ); + RISCV_WRITEX ( q, uint64_t, "d" ); + #else + RISCV_READX ( q, uint64_t, "d" ); + RISCV_WRITEX ( q, uint64_t, "d" ); + #endif +#else + RISCV_READX_FUSED ( q, uint64_t, "w" ); + RISCV_WRITEX_FUSED ( q, uint64_t, "w" ); +#endif + +/* + * Memory barrier + * + */ +static inline __always_inline void +IOAPI_INLINE ( riscv, mb ) ( void ) { + __asm__ __volatile__ ( "fence" : : : "memory" ); +} + +/* Dummy PIO */ +DUMMY_PIO ( riscv ); + +#endif /* _IPXE_RISCV_IO_H */ diff --git a/src/arch/riscv/include/ipxe/sbi.h b/src/arch/riscv/include/ipxe/sbi.h new file mode 100644 index 000000000..4364098b9 --- /dev/null +++ b/src/arch/riscv/include/ipxe/sbi.h @@ -0,0 +1,213 @@ +#ifndef _IPXE_SBI_H +#define _IPXE_SBI_H + +/** @file + * + * Supervisor Binary Interface (SBI) + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> + +/** An SBI function return value */ +struct sbi_return { + /** Error status (returned in a0) */ + long error; + /** Data value (returned in a1) */ + long value; +}; + +/** + * @defgroup sbierrors SBI errors + * + * *{ + */ +#define SBI_SUCCESS 0 /**< Completed successfully */ +#define SBI_ERR_FAILED -1 /**< Failed */ +#define SBI_ERR_NOT_SUPPORTED -2 /**< Not supported */ +#define SBI_ERR_INVALID_PARAM -3 /**< Invalid parameter(s) */ +#define SBI_ERR_DENIED -4 /**< Denied or not allowed */ +#define SBI_ERR_INVALID_ADDRESS -5 /**< Invalid address(es) */ +#define SBI_ERR_ALREADY_AVAILABLE -6 /**< Already available */ +#define SBI_ERR_ALREADY_STARTED -7 /**< Already started */ +#define SBI_ERR_ALREADY_STOPPED -8 /**< Already stopped */ +#define SBI_ERR_NO_SHMEM -9 /**< Shared memory not available */ +#define SBI_ERR_INVALID_STATE -10 /**< Invalid state */ +#define SBI_ERR_BAD_RANGE -11 /**< Bad (or invalid) range */ +#define SBI_ERR_TIMEOUT -12 /**< Failed due to timeout */ +#define SBI_ERR_IO -13 /**< Input/output error */ +/** @} */ + +/** Construct SBI extension ID */ +#define SBI_EID( c1, c2, c3, c4 ) \ + ( (int) ( ( (c1) << 24 ) | ( (c2) << 16 ) | ( (c3) << 8 ) | (c4) ) ) + +/** + * Call supervisor with no parameters + * + * @v eid Extension ID + * @v fid Function ID + * @ret ret Return value + */ +static inline __attribute__ (( always_inline )) struct sbi_return +sbi_ecall_0 ( int eid, int fid ) { + register unsigned long a7 asm ( "a7" ) = ( ( long ) eid ); + register unsigned long a6 asm ( "a6" ) = ( ( long ) fid ); + register unsigned long a0 asm ( "a0" ); + register unsigned long a1 asm ( "a1" ); + struct sbi_return ret; + + __asm__ __volatile__ ( "ecall" + : "=r" ( a0 ), "=r" ( a1 ) + : "r" ( a6 ), "r" ( a7 ) + : "memory" ); + ret.error = a0; + ret.value = a1; + return ret; +} + +/** + * Call supervisor with one parameter + * + * @v eid Extension ID + * @v fid Function ID + * @v p0 Parameter 0 + * @ret ret Return value + */ +static inline __attribute__ (( always_inline )) struct sbi_return +sbi_ecall_1 ( int eid, int fid, unsigned long p0 ) { + register unsigned long a7 asm ( "a7" ) = ( ( long ) eid ); + register unsigned long a6 asm ( "a6" ) = ( ( long ) fid ); + register unsigned long a0 asm ( "a0" ) = p0; + register unsigned long a1 asm ( "a1" ); + struct sbi_return ret; + + __asm__ __volatile__ ( "ecall" + : "+r" ( a0 ), "=r" ( a1 ) + : "r" ( a6 ), "r" ( a7 ) + : "memory" ); + ret.error = a0; + ret.value = a1; + return ret; +} + +/** + * Call supervisor with two parameters + * + * @v eid Extension ID + * @v fid Function ID + * @v p0 Parameter 0 + * @v p1 Parameter 1 + * @ret ret Return value + */ +static inline __attribute__ (( always_inline )) struct sbi_return +sbi_ecall_2 ( int eid, int fid, unsigned long p0, unsigned long p1 ) { + register unsigned long a7 asm ( "a7" ) = ( ( long ) eid ); + register unsigned long a6 asm ( "a6" ) = ( ( long ) fid ); + register unsigned long a0 asm ( "a0" ) = p0; + register unsigned long a1 asm ( "a1" ) = p1; + struct sbi_return ret; + + __asm__ __volatile__ ( "ecall" + : "+r" ( a0 ), "+r" ( a1 ) + : "r" ( a6 ), "r" ( a7 ) + : "memory" ); + ret.error = a0; + ret.value = a1; + return ret; +} + +/** + * Call supervisor with three parameters + * + * @v eid Extension ID + * @v fid Function ID + * @v p0 Parameter 0 + * @v p1 Parameter 1 + * @v p2 Parameter 2 + * @ret ret Return value + */ +static inline __attribute__ (( always_inline )) struct sbi_return +sbi_ecall_3 ( int eid, int fid, unsigned long p0, unsigned long p1, + unsigned long p2 ) { + register unsigned long a7 asm ( "a7" ) = ( ( long ) eid ); + register unsigned long a6 asm ( "a6" ) = ( ( long ) fid ); + register unsigned long a0 asm ( "a0" ) = p0; + register unsigned long a1 asm ( "a1" ) = p1; + register unsigned long a2 asm ( "a2" ) = p2; + struct sbi_return ret; + + __asm__ __volatile__ ( "ecall" + : "+r" ( a0 ), "+r" ( a1 ) + : "r" ( a2 ), "r" ( a6 ), "r" ( a7 ) + : "memory" ); + ret.error = a0; + ret.value = a1; + return ret; +} + +/** + * Call supervisor with no parameters + * + * @v fid Legacy function ID + * @ret ret Return value + */ +static inline __attribute__ (( always_inline )) long +sbi_legacy_ecall_0 ( int fid ) { + register unsigned long a7 asm ( "a7" ) = ( ( long ) fid ); + register unsigned long a0 asm ( "a0" ); + + __asm__ __volatile__ ( "ecall" + : "=r" ( a0 ) + : "r" ( a7 ) + : "memory" ); + return a0; +} + +/** + * Call supervisor with one parameter + * + * @v fid Legacy function ID + * @v p0 Parameter 0 + * @ret ret Return value + */ +static inline __attribute__ (( always_inline )) long +sbi_legacy_ecall_1 ( int fid, unsigned long p0 ) { + register unsigned long a7 asm ( "a7" ) = ( ( long ) fid ); + register unsigned long a0 asm ( "a0" ) = p0; + + __asm__ __volatile__ ( "ecall" + : "+r" ( a0 ) + : "r" ( a7 ) + : "memory" ); + return a0; +} + +/** Convert an SBI error code to an iPXE status code */ +#define ESBI( error ) EPLATFORM ( EINFO_EPLATFORM, error ) + +/** Legacy extensions */ +#define SBI_LEGACY_PUTCHAR 0x01 /**< Console Put Character */ +#define SBI_LEGACY_GETCHAR 0x02 /**< Console Get Character */ +#define SBI_LEGACY_SHUTDOWN 0x08 /**< System Shutdown */ + +/** Base extension */ +#define SBI_BASE 0x10 +#define SBI_BASE_MVENDORID 0x04 /**< Get machine vendor ID */ + +/** System reset extension */ +#define SBI_SRST SBI_EID ( 'S', 'R', 'S', 'T' ) +#define SBI_SRST_SYSTEM_RESET 0x00 /**< Reset system */ +#define SBI_RESET_SHUTDOWN 0x00000000 /**< Shutdown */ +#define SBI_RESET_COLD 0x00000001 /**< Cold reboot */ +#define SBI_RESET_WARM 0x00000002 /**< Warm reboot */ + +/** Debug console extension */ +#define SBI_DBCN SBI_EID ( 'D', 'B', 'C', 'N' ) +#define SBI_DBCN_WRITE 0x00 /**< Console Write */ +#define SBI_DBCN_READ 0x01 /**< Console Read */ +#define SBI_DBCN_WRITE_BYTE 0x02 /**< Console Write Byte */ + +#endif /* _IPXE_SBI_H */ diff --git a/src/arch/riscv/include/ipxe/sbi_reboot.h b/src/arch/riscv/include/ipxe/sbi_reboot.h new file mode 100644 index 000000000..e8d6e82bf --- /dev/null +++ b/src/arch/riscv/include/ipxe/sbi_reboot.h @@ -0,0 +1,18 @@ +#ifndef _IPXE_BIOS_REBOOT_H +#define _IPXE_BIOS_REBOOT_H + +/** @file + * + * Supervisor Binary Interface (SBI) reboot mechanism + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#ifdef REBOOT_SBI +#define REBOOT_PREFIX_sbi +#else +#define REBOOT_PREFIX_sbi __sbi_ +#endif + +#endif /* _IPXE_BIOS_REBOOT_H */ diff --git a/src/arch/riscv/include/ipxe/svpage.h b/src/arch/riscv/include/ipxe/svpage.h new file mode 100644 index 000000000..897a3379a --- /dev/null +++ b/src/arch/riscv/include/ipxe/svpage.h @@ -0,0 +1,28 @@ +#ifndef _IPXE_SVPAGE_H +#define _IPXE_SVPAGE_H + +/** @file + * + * Supervisor page table management + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> + +#ifdef IOMAP_SVPAGE +#define IOMAP_PREFIX_svpage +#else +#define IOMAP_PREFIX_svpage __svpage_ +#endif + +static inline __always_inline unsigned long +IOMAP_INLINE ( svpage, io_to_bus ) ( volatile const void *io_addr ) { + /* Not easy to do; just return the CPU address for debugging purposes */ + return ( ( intptr_t ) io_addr ); +} + +extern void * svpage_dma32 ( void ); + +#endif /* _IPXE_SVPAGE_H */ diff --git a/src/arch/riscv/include/ipxe/xthead.h b/src/arch/riscv/include/ipxe/xthead.h new file mode 100644 index 000000000..d0c9449ef --- /dev/null +++ b/src/arch/riscv/include/ipxe/xthead.h @@ -0,0 +1,21 @@ +#ifndef _IPXE_XTHEAD_H +#define _IPXE_XTHEAD_H + +/** @file + * + * T-Head vendor extensions + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** T-Head machine vendor ID */ +#define THEAD_MVENDORID 0x5b7 + +/** T-Head SXSTATUS CSR */ +#define THEAD_SXSTATUS 0x5c0 +#define THEAD_SXSTATUS_THEADISAEE 0x00400000 /**< General ISA extensions */ + +extern int xthead_supported ( unsigned long feature ); + +#endif /* _IPXE_XTHEAD_H */ diff --git a/src/arch/riscv/include/ipxe/zicbom.h b/src/arch/riscv/include/ipxe/zicbom.h new file mode 100644 index 000000000..4ba165f3c --- /dev/null +++ b/src/arch/riscv/include/ipxe/zicbom.h @@ -0,0 +1,17 @@ +#ifndef _IPXE_ZICBOM_H +#define _IPXE_ZICBOM_H + +/** @file + * + * Cache-block management operations (Zicbom) + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> + +extern void cache_clean ( const void *start, size_t len ); +extern void cache_invalidate ( void *start, size_t len ); + +#endif /* _IPXE_ZICBOM_H */ diff --git a/src/arch/riscv/interface/sbi/sbi_console.c b/src/arch/riscv/interface/sbi/sbi_console.c new file mode 100644 index 000000000..1c3784ec8 --- /dev/null +++ b/src/arch/riscv/interface/sbi/sbi_console.c @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +/** @file + * + * SBI debug console + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <ipxe/sbi.h> +#include <ipxe/io.h> +#include <ipxe/keys.h> +#include <ipxe/serial.h> +#include <ipxe/console.h> +#include <config/console.h> + +/* Set default console usage if applicable */ +#if ! ( defined ( CONSOLE_SBI ) && CONSOLE_EXPLICIT ( CONSOLE_SBI ) ) +#undef CONSOLE_SBI +#define CONSOLE_SBI ( CONSOLE_USAGE_ALL & ~CONSOLE_USAGE_LOG ) +#endif + +extern void early_uart_putchar ( int character ); + +/** Dummy serial console (if not present in build) */ +struct uart *serial_console __attribute__ (( weak )); + +/** Buffered input character (if any) */ +static unsigned char sbi_console_input; + +/** + * Print a character to SBI console + * + * @v character Character to be printed + */ +static void sbi_putchar ( int character ) { + struct sbi_return ret; + + /* Do nothing if a real serial console has been enabled */ + if ( serial_console ) + return; + + /* Write byte to early UART, if enabled */ + early_uart_putchar ( character ); + + /* Write byte to console */ + ret = sbi_ecall_1 ( SBI_DBCN, SBI_DBCN_WRITE_BYTE, character ); + if ( ! ret.error ) + return; + + /* Debug extension not supported: try legacy method */ + sbi_legacy_ecall_1 ( SBI_LEGACY_PUTCHAR, character ); +} + +/** + * Get character from SBI console + * + * @ret character Character read from console, if any + */ +static int sbi_getchar ( void ) { + int character; + + /* Consume and return buffered character, if any */ + character = sbi_console_input; + sbi_console_input = 0; + + /* Convert DEL to backspace */ + if ( character == DEL ) + character = BACKSPACE; + + return character; +} + +/** + * Check for character ready to read from SBI console + * + * @ret True Character available to read + * @ret False No character available to read + */ +static int sbi_iskey ( void ) { + struct sbi_return ret; + long key; + + /* Do nothing if we already have a buffered character */ + if ( sbi_console_input ) + return sbi_console_input; + + /* Do nothing if a real serial console has been enabled */ + if ( serial_console ) + return 0; + + /* Read and buffer byte from console, if any */ + ret = sbi_ecall_3 ( SBI_DBCN, SBI_DBCN_READ, + sizeof ( sbi_console_input ), + virt_to_phys ( &sbi_console_input ), 0 ); + if ( ! ret.error ) + return ret.value; + + /* Debug extension not supported: try legacy method */ + key = sbi_legacy_ecall_0 ( SBI_LEGACY_GETCHAR ); + if ( key > 0 ) { + sbi_console_input = key; + return key; + } + + /* No character available */ + return 0; +} + +/** SBI console */ +struct console_driver sbi_console_driver __console_driver = { + .putchar = sbi_putchar, + .getchar = sbi_getchar, + .iskey = sbi_iskey, + .usage = CONSOLE_SBI, +}; diff --git a/src/arch/riscv/interface/sbi/sbi_reboot.c b/src/arch/riscv/interface/sbi/sbi_reboot.c new file mode 100644 index 000000000..b1c742ec7 --- /dev/null +++ b/src/arch/riscv/interface/sbi/sbi_reboot.c @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/** @file + * + * Supervisor Binary Interface (SBI) reboot mechanism + * + */ + +#include <errno.h> +#include <string.h> +#include <ipxe/sbi.h> +#include <ipxe/reboot.h> + +/** + * Reboot system + * + * @v flags Reboot flags + */ +static void sbi_reboot ( int flags ) { + struct sbi_return ret; + int warm; + int rc; + + /* Reboot system */ + warm = ( flags & REBOOT_WARM ); + ret = sbi_ecall_2 ( SBI_SRST, SBI_SRST_SYSTEM_RESET, + ( warm ? SBI_RESET_WARM : SBI_RESET_COLD ), 0 ); + + /* Any return is an error */ + rc = -ESBI ( ret.error ); + DBGC ( SBI_SRST, "SBI %s reset failed: %s\n", + ( warm ? "warm" : "cold" ), strerror ( rc ) ); + + /* Try a legacy shutdown */ + sbi_legacy_ecall_0 ( SBI_LEGACY_SHUTDOWN ); + DBGC ( SBI_SRST, "SBI legacy shutdown failed\n" ); +} + +/** + * Power off system + * + * @ret rc Return status code + */ +static int sbi_poweroff ( void ) { + struct sbi_return ret; + int rc; + + /* Shut down system */ + ret = sbi_ecall_2 ( SBI_SRST, SBI_SRST_SYSTEM_RESET, + SBI_RESET_SHUTDOWN, 0 ); + + /* Any return is an error */ + rc = -ESBI ( ret.error ); + DBGC ( SBI_SRST, "SBI shutdown failed: %s\n", strerror ( rc ) ); + + /* Try a legacy shutdown */ + sbi_legacy_ecall_0 ( SBI_LEGACY_SHUTDOWN ); + DBGC ( SBI_SRST, "SBI legacy shutdown failed\n" ); + + return rc; +} + +PROVIDE_REBOOT ( sbi, reboot, sbi_reboot ); +PROVIDE_REBOOT ( sbi, poweroff, sbi_poweroff ); diff --git a/src/arch/riscv/prefix/libprefix.S b/src/arch/riscv/prefix/libprefix.S new file mode 100644 index 000000000..338131103 --- /dev/null +++ b/src/arch/riscv/prefix/libprefix.S @@ -0,0 +1,1529 @@ +/* + * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * RISC-V prefix library + * + */ + +#include <config/serial.h> +#include <config/fault.h> + + .section ".note.GNU-stack", "", @progbits + .text + + /* Link-time base address of _prefix + * + * This will be not be updated if runtime relocations are applied. + */ + .section ".rodata.prefix_link", "a", @progbits + .balign ( __riscv_xlen / 8 ) +prefix_link: + .dword _base + .size prefix_link, . - prefix_link + + /* Virtual address of _prefix + * + * This will be updated if runtime relocations are applied. + */ + .section ".rodata.prefix_virt", "a", @progbits + .balign ( __riscv_xlen / 8 ) +prefix_virt: + .dword _prefix + .size prefix_virt, . - prefix_virt + +/***************************************************************************** + * + * Print character via debug console extension + * + ***************************************************************************** + * + * Print a single character via the SBI DBCN extension. + * + * Parameters: + * + * a0 - Character to print + * + * Returns: + * + * a0 - Zero if character printed successfully + * a1 - Overwritten + * a6 - Overwritten + * a7 - Overwritten + * + */ + +/* SBI debug console extension */ +#define SBI_DBCN ( ( 'D' << 24 ) | ( 'B' << 16 ) | ( 'C' << 8 ) | 'N' ) +#define SBI_DBCN_WRITE_BYTE 0x02 + + .macro print_char_dbcn + li a7, SBI_DBCN + li a6, SBI_DBCN_WRITE_BYTE + ecall + .endm + +/***************************************************************************** + * + * Print character via legacy extension + * + ***************************************************************************** + * + * Print a single character via the SBI putchar legacy extension. + * + * Parameters: + * + * a0 - Character to print + * + * Returns: + * + * a0 - Overwritten + * a7 - Overwritten + * + */ + +/* SBI legacy console putchar */ +#define SBI_LEGACY_PUTCHAR 0x01 + + .macro print_char_legacy + li a7, SBI_LEGACY_PUTCHAR + ecall + .endm + +/***************************************************************************** + * + * Print character via early UART + * + ***************************************************************************** + * + * Print a single character via a UART. + * + * For devices without a functional SBI console, a UART at a hardcoded + * address can be used as a last resort mechanism for obtaining debug + * output from the prefix. + * + * Parameters: + * + * a0 - Character to print + * + * Returns: + * + * a0 - Preserved + * a1 - May be overwritten + * a6 - May be overwritten + * a7 - May be overwritten + * + */ + +/* Default to no UART, if not specified */ +#ifndef EARLY_UART_MODEL +#define EARLY_UART_MODEL none +#endif + +/* Default to a register shift of zero, if not specified */ +#ifndef EARLY_UART_REG_SHIFT +#define EARLY_UART_REG_SHIFT 0 +#endif + +#define print_char_uart _C2 ( print_char_uart_, EARLY_UART_MODEL ) + +#define early_uart_reg_base _C2 ( early_uart_reg_base_, __riscv_xlen ) + + /* Print character via nonexistent UART */ + .macro print_char_uart_none + .endm + + /* + * Get UART base address (64-bit addressing) + */ + .macro early_uart_reg_base_64 reg + csrr \reg, satp + beqz \reg, early_uart_reg_base_64_nonpaged_\@ + LOADN \reg, early_uart_reg_base_64_virt + j early_uart_reg_base_64_done_\@ +early_uart_reg_base_64_nonpaged_\@: + li \reg, EARLY_UART_REG_BASE +early_uart_reg_base_64_done_\@: + .endm + + /* + * Get UART base address (32-bit addressing) + */ + .macro early_uart_reg_base_32 reg + li \reg, EARLY_UART_REG_BASE + sub \reg, \reg, tp + .endm + +/***************************************************************************** + * + * Print character via 8250-compatible early UART + * + ***************************************************************************** + * + * Print a single character via an 8250- or 16550-compatible UART. + * + * Parameters: + * + * a0 - Character to print + * + * Returns: + * + * a0 - Preserved + * a1 - Overwritten + * a7 - Overwritten + * + */ + +/* 8250-compatible UART transmit registers */ +#define EARLY_UART_8250_TX ( 0 << EARLY_UART_REG_SHIFT ) +#define EARLY_UART_8250_LSR ( 5 << EARLY_UART_REG_SHIFT ) +#define EARLY_UART_8250_LSR_THRE 0x20 + + .macro print_char_uart_8250 + early_uart_reg_base a7 + sb a0, EARLY_UART_8250_TX(a7) + fence +early_uart_8250_wait_\@: + lbu a1, EARLY_UART_8250_LSR(a7) + andi a1, a1, EARLY_UART_8250_LSR_THRE + beqz a1, early_uart_8250_wait_\@ + .endm + +/***************************************************************************** + * + * Print character via SiFive-compatible early UART + * + ***************************************************************************** + * + * Print a single character via a SiFive-compatible UART. + * + * Parameters: + * + * a0 - Character to print + * + * Returns: + * + * a0 - Preserved + * a1 - Overwritten + * a7 - Overwritten + * + */ + +/* SiFive-compatible UART transmit registers */ +#define EARLY_UART_SIFIVE_TXFIFO ( 0 << EARLY_UART_REG_SHIFT ) + + .macro print_char_uart_sifive + early_uart_reg_base a7 + sw a0, EARLY_UART_SIFIVE_TXFIFO(a7) + fence +early_uart_sifive_wait_\@: + lw a1, EARLY_UART_SIFIVE_TXFIFO(a7) + bltz a1, early_uart_sifive_wait_\@ + .endm + +/***************************************************************************** + * + * Print single character to early UART (from C code) + * + ***************************************************************************** + * + * This function is called by the SBI console driver to output a + * character to the early UART (if enabled). + * + * The standard C ABI applies to this function. + * + * Parameters: + * + * a0 - Character to print + * + * Returns: none + * + */ + + .section ".prefix.early_uart_putchar", "ax", @progbits + .globl early_uart_putchar +early_uart_putchar: + print_char_uart + ret + .size early_uart_putchar, . - early_uart_putchar + +/***************************************************************************** + * + * Print message to debug console + * + ***************************************************************************** + * + * Print a NUL-terminated string to the debug console. + * + * This function prints one character at a time via the "write byte" + * call (rather than using "write string"), since this avoids any need + * to know the current virtual-physical address translation. It does + * not require a valid stack. + * + * Note that the parameter is passed in register t1 (rather than a0) + * and all non-temporary registers are preserved. + * + * Parameters: + * + * t1 - Pointer to string + * + * Returns: none + * + */ + + .section ".prefix.print_message", "ax", @progbits + .globl print_message +print_message: + /* Handle alternate link register */ + mv t0, ra +print_message_alt: + /* Register usage: + * + * a0 - current character + * t0 - alternate link register + * t1 - character pointer + * t2 - preserved a0 + * t3 - preserved a1 + * t4 - preserved a6 + * t5 - preserved a7 + */ + mv t2, a0 + mv t3, a1 + mv t4, a6 + mv t5, a7 + +1: /* Print each character in turn */ + lbu a0, (t1) + addi t1, t1, 1 + beqz a0, 2f + print_char_uart + print_char_dbcn + beqz a0, 1b + lbu a0, -1(t1) + print_char_legacy + j 1b +2: + /* Restore registers and return (via alternate link register) */ + mv a7, t5 + mv a6, t4 + mv a1, t3 + mv a0, t2 + jr t0 + .size print_message, . - print_message + + /* + * Display progress message (if debugging is enabled) + */ + .macro progress message +#ifndef NDEBUG + .section ".rodata.progress_\@", "a", @progbits +progress_\@: + .asciz "\message" + .size progress_\@, . - progress_\@ + .previous + la t1, progress_\@ + jal t0, print_message_alt +#endif + .endm + +/***************************************************************************** + * + * Print hexadecimal value to debug console + * + ***************************************************************************** + * + * Print a register value in hexadecimal to the debug console. + * + * This function does not require a valid stack. + * + * Note that the parameters are passed in registers t1 and t2 (rather + * than a0) and all non-temporary registers are preserved. + * + * Parameters: + * + * t1 - Value to print + * t2 - Number of bits to print (must be a multiple of 4) + * + * Returns: none + * + */ + + /* + * Convert a single nibble to an ASCII character + */ + .macro nibble_to_ascii reg + addi \reg, \reg, -10 + bltz \reg, dec_\@ + addi \reg, \reg, ( 'a' - ( '0' + 10 ) ) +dec_\@: addi \reg, \reg, ( '0' + 10 ) + .endm + + .section ".prefix.print_hex_value", "ax", @progbits + .globl print_hex_value +print_hex_value: + /* Handle alternate link register */ + mv t0, ra +print_hex_value_alt: + /* Register usage: + * + * a0 - current digit / general temporary + * t0 - alternate link register + * t1 - current value + * t2 - digit counter + * t3 - preserved a0 + * t4 - preserved a1 + * t5 - preserved a6 + * t6 - preserved a7 + */ + mv t3, a0 + mv t4, a1 + mv t5, a6 + mv t6, a7 + + /* Skip any unprinted digits */ + li a0, __riscv_xlen + sub a0, a0, t2 + sll t1, t1, a0 + +1: /* Print each digit in turn */ + srli a0, t1, ( __riscv_xlen - 4 ) + nibble_to_ascii a0 + print_char_uart + print_char_dbcn + beqz a0, 2f + srli a0, t1, ( __riscv_xlen - 4 ) + nibble_to_ascii a0 + print_char_legacy +2: slli t1, t1, 4 + addi t2, t2, -4 + bgtz t2, 1b + + /* Restore registers and return (via alternate link register) */ + mv a7, t6 + mv a6, t5 + mv a1, t4 + mv a0, t3 + jr t0 + .size print_hex_value, . - print_hex_value + + /* + * Display hexadecimal register value (if debugging is enabled) + */ + .macro print_hex_reg reg, bits=__riscv_xlen +#ifndef NDEBUG + mv t1, \reg + li t2, \bits + jal t0, print_hex_value_alt +#endif + .endm + + /* + * Display hexadecimal symbol address (if debugging is enabled) + */ + .macro print_hex_addr sym +#ifndef NDEBUG + la t1, \sym + li t2, __riscv_xlen + jal t0, print_hex_value_alt +#endif + .endm + + /* + * Display hexadecimal data value (if debugging is enabled) + */ + .macro print_hex_data sym +#ifndef NDEBUG + LOADN t1, \sym + li t2, __riscv_xlen + jal t0, print_hex_value_alt +#endif + .endm + +/***************************************************************************** + * + * Apply compressed relocation records + * + ***************************************************************************** + * + * Apply compressed relocation records to fix up iPXE to run at its + * current virtual address. + * + * This function must run before .bss is zeroed (since the relocation + * records are overlaid with .bss). It does not require a valid stack + * pointer. + * + * Parameters: none + * + * a0 - Relocation records + * + * Returns: none + * + */ + +/** Number of bits in a skip value */ +#define ZREL_SKIP_BITS 19 + + .section ".prefix.apply_relocs", "ax", @progbits + .globl apply_relocs +apply_relocs: + /* Register usage: + * + * a0 - current relocation record pointer + * a1 - current relocation target address + * a2 - relocation addend + * a3 - current relocation record value + * a4 - number of bits remaining in current relocation record + */ + la a1, _prefix + + /* Calculate relocation addend */ + LOADN a2, prefix_virt + sub a2, a1, a2 + + /* Skip applying relocations if addend is zero */ + beqz a2, apply_relocs_done + progress " reloc" + + /* Test writability + * + * We do this to avoid accidentally sending an undefined + * sequence of commands to a flash device, if we are started + * from read-only memory with no paging support. + * + * We attempt to write an all-ones pattern, on the basis that + * this pattern will harmlessly cause any flash device + * conforming to the CFI01 specification to enter the default + * "read array" state. + */ + la t0, apply_relocs_test + li t1, -1 + STOREN t1, (t0) + LOADN t2, (t0) + bne t1, t2, apply_relocs_failed + +apply_relocs_loop: + /* Read new relocation record */ + LOADN a3, (a0) + addi a0, a0, ( __riscv_xlen / 8 ) + li a4, ( __riscv_xlen - 1 ) + + /* Consume and apply skip, if present (i.e. if MSB=0) */ + bltz a3, 1f + addi a4, a4, -ZREL_SKIP_BITS + srli t0, a3, ( __riscv_xlen - ( ZREL_SKIP_BITS + 1 ) ) + slli t0, t0, ( ( __riscv_xlen / 32 ) + 1 ) + add a1, a1, t0 +1: + /* Apply relocations corresponding to set bits in record */ +1: andi t0, a3, 1 + beqz t0, 2f + LOADN t1, (a1) + add t1, t1, a2 + STOREN t1, (a1) +2: addi a1, a1, ( __riscv_xlen / 8 ) + srli a3, a3, 1 + addi a4, a4, -1 + bnez a4, 1b + + /* Loop until we have reached a terminator record (MSB=0, offset=0) */ + bnez a3, apply_relocs_loop + + /* Check that relocations were applied successfully */ + la t0, _prefix + LOADN t1, prefix_virt + bne t0, t1, apply_relocs_failed + +apply_relocs_done: + /* Return to caller */ + progress " ok\r\n" + ret + +apply_relocs_failed: + /* Failure to apply relocations (if relocations were needed) + * is a fatal error. + */ + progress " failed\r\n" + j reset_system + .size apply_relocs, . - apply_relocs + + /* Writability test + * + * Placed within .data rather than .bss, since we need this to + * be within the range of the stored iPXE image. + */ + .section ".data.apply_relocs_test", "aw", @progbits + .balign ( __riscv_xlen / 8 ) +apply_relocs_test: + .space ( __riscv_xlen / 8 ) + .size apply_relocs_test, . - apply_relocs_test + +/***************************************************************************** + * + * Enable paging + * + ***************************************************************************** + * + * This function must be called with flat physical addressing. It + * does not require a valid stack pointer. + * + * Parameters: + * + * a0 - Page table to fill in (4kB, must be aligned to a 4kB boundary) + * + * Returns: + * + * a0 - Size of accessible physical address space (or zero for no limit) + * tp - Virtual address offset + * pc - Updated to a virtual address if paging enabled + * + */ + +/** Number of bits in a page offset */ +#define PAGE_SHIFT 12 + +/** Page size */ +#define PAGE_SIZE ( 1 << PAGE_SHIFT ) + +/** Size of a page table entry (log2) */ +#define PTE_SIZE_LOG2 ( ( __riscv_xlen / 32 ) + 1 ) + +/** Size of a page table entry */ +#define PTE_SIZE ( 1 << PTE_SIZE_LOG2 ) + +/** Number of page table entries (log2) */ +#define PTE_COUNT_LOG2 ( PAGE_SHIFT - PTE_SIZE_LOG2 ) + +/** Number of page table entries */ +#define PTE_COUNT ( 1 << PTE_COUNT_LOG2 ) + +/** Number of bits in a virtual or physical page number */ +#define VPPN_SHIFT PTE_COUNT_LOG2 + +/* Page table entry flags */ +#define PTE_V 0x00000001 /**< Page table entry is valid */ +#define PTE_R 0x00000002 /**< Page is readable */ +#define PTE_W 0x00000004 /**< Page is writable */ +#define PTE_X 0x00000008 /**< Page is executable */ +#define PTE_A 0x00000040 /**< Page has been accessed */ +#define PTE_D 0x00000080 /**< Page is dirty */ + +/* Page table entry flags for our leaf pages */ +#define PTE_LEAF ( PTE_D | PTE_A | PTE_X | PTE_W | PTE_R | PTE_V ) + +/** Physical page number LSB in PTE */ +#define PTE_PPN_LSB(x) ( 10 + (x) * VPPN_SHIFT ) +#define PTE_PPN4_LSB PTE_PPN_LSB(4) /**< PPN[4] LSB (Sv57) */ +#define PTE_PPN3_LSB PTE_PPN_LSB(3) /**< PPN[3] LSB (Sv57 & Sv48) */ +#define PTE_PPN2_LSB PTE_PPN_LSB(2) /**< PPN[2] LSB (Sv57, Sv48, & Sv39) */ +#define PTE_PPN1_LSB PTE_PPN_LSB(1) /**< PPN[1] LSB (all levels) */ +#define PTE_PPN0_LSB PTE_PPN_LSB(0) /**< PPN[0] LSB (all levels) */ + +/** Page table entry physical page address shift */ +#define PTE_PPN_SHIFT ( PAGE_SHIFT - PTE_PPN0_LSB ) + +/** Virtual page number LSB */ +#define VPN_LSB(x) ( PAGE_SHIFT + (x) * VPPN_SHIFT ) +#define VPN4_LSB VPN_LSB(4) /**< VPN[4] LSB (Sv57) */ +#define VPN3_LSB VPN_LSB(3) /**< VPN[3] LSB (Sv57 & Sv48) */ +#define VPN2_LSB VPN_LSB(2) /**< VPN[2] LSB (Sv57, Sv48, & Sv39) */ +#define VPN1_LSB VPN_LSB(1) /**< VPN[1] LSB (all levels) */ +#define VPN0_LSB VPN_LSB(0) /**< VPN[0] LSB (all levels) */ + +/* Paging modes */ +#define SATP_MODE_SV57 10 /**< Five-level paging (Sv57) */ +#define SATP_MODE_SV48 9 /**< Four-level paging (Sv48) */ +#define SATP_MODE_SV39 8 /**< Three-level paging (Sv39) */ +#define SATP_MODE_SV32 1 /**< Two-level paging (Sv32) */ + +/** Paging mode shift */ +#if __riscv_xlen == 64 +#define SATP_MODE_SHIFT 60 +#else +#define SATP_MODE_SHIFT 31 +#endif + + .globl enable_paging + .equ enable_paging, _C2 ( enable_paging_, __riscv_xlen ) + + /* Paging mode names (for debug messages) */ + .section ".rodata.paging_mode_names", "a", @progbits +paging_mode_names: + .asciz "none" + .org ( paging_mode_names + 5 * SATP_MODE_SV32 ) + .asciz "Sv32" + .org ( paging_mode_names + 5 * SATP_MODE_SV39 ) + .asciz "Sv39" + .org ( paging_mode_names + 5 * SATP_MODE_SV48 ) + .asciz "Sv48" + .org ( paging_mode_names + 5 * SATP_MODE_SV57 ) + .asciz "Sv57" + .size paging_mode_names, . - paging_mode_names + + /* + * Display paging mode name (if debugging is enabled) + */ + .macro paging_mode_name reg +#ifndef NDEBUG + slli t0, \reg, 2 + add t0, t0, \reg + la t1, paging_mode_names + add t1, t1, t0 + jal t0, print_message_alt +#endif + .endm + + /* Maximum physical alignment + * + * We align to a "megapage" boundary to simplify the task of + * setting up page table mappings. + */ + .globl _max_align + .equ _max_align, ( 1 << VPN1_LSB ) + + /* Space for page table + * + * This can be used only once .bss is known to be writable. + */ + .section ".bss.page_table", "a", @nobits + .globl page_table + .balign PAGE_SIZE +page_table: + .space PAGE_SIZE + .size page_table, . - page_table + + /* Convert physical address to virtual address */ + .macro phys_to_virt rd, rs:vararg + _C2 ( phys_to_virt_, __riscv_xlen ) \rd, \rs + .endm + +/***************************************************************************** + * + * Disable paging + * + ***************************************************************************** + * + * This function may be called with either virtual or flat physical + * addressing. It does not require a valid stack pointer. + * + * Parameters: + * + * tp - Virtual address offset + * + * Returns: + * + * tp - Virtual address offset (zeroed) + * pc - Updated to a physical address + * + */ + + .globl disable_paging + .equ disable_paging, _C2 ( disable_paging_, __riscv_xlen ) + +/***************************************************************************** + * + * Enable 64-bit paging + * + ***************************************************************************** + * + * Construct a 64-bit page table to identity-map the whole of the + * mappable physical address space, and to map iPXE itself at its + * link-time address (which must be 2MB-aligned and be within the + * upper half of the kernel address space). + * + * This function must be called with flat physical addressing. It + * does not require a valid stack pointer. + * + * Parameters: + * + * a0 - Page table to fill in (4kB, must be aligned to a 4kB boundary) + * + * Returns: + * + * a0 - Size of accessible physical address space (or zero for no limit) + * tp - Virtual address offset + * pc - Updated to a virtual address if paging enabled + * + * A 4kB 64-bit page table contains 512 8-byte PTEs. We choose to use + * these as: + * + * - PTE[0-255] : Identity map for the physical address space. + * + * This conveniently requires exactly 256 PTEs, regardless of the + * paging level. Higher paging levels are able to identity-map a + * larger physical address space: + * + * Sv57 : 256 x 256TB "petapages" (55-bit physical address space) + * Sv48 : 256 x 512GB "terapages" (46-bit physical address space) + * Sv39 : 256 x 1GB "gigapages" (37-bit physical address space) + * + * Note that Sv48 and Sv39 cannot identity-map the whole of the + * available physical address space, since the virtual address + * space is not large enough (and is halved by the constraint + * that virtual addresses with bit 47/38 set must also have all + * higher bits set, and so cannot identity-map to a 55-bit + * physical address). + * + * - PTE[x-y] : Virtual address map for iPXE + * + * These are 2MB "megapages" used to map the link-time virtual + * address range used by iPXE itself. We can use any 2MB-aligned + * range within 0xffffffffe0800000-0xffffffffffc00000, which + * breaks down as: + * + * VPN[4] = 511 (in Sv57, must be all-ones in Sv48 and Sv39) + * VPN[3] = 511 (in Sv57 and Sv48, must be all-ones in Sv39) + * VPN[2] = 511 (in all paging levels) + * VPN[1] = 260-510 (in all paging levels) + * VPN[0] = 0 (in all paging levels) + * + * In most builds, only a single 2MB "megapage" will be needed. + * We choose a link-time starting address of 0xffffffffeb000000 + * within the permitted range, since the "eb" pattern is fairly + * distinctive and so makes it easy to visually identify any + * addresses originating from within iPXE's virtual address + * space. + * + * - PTE[511] : Recursive next level page table pointer + * + * This is a non-leaf PTE that points back to the page table + * itself. It acts as the next level page table pointer for: + * + * VPN[4] = 511 (in Sv57) + * VPN[3] = 511 (in Sv57 and Sv48) + * VPN[2] = 511 (in Sv57, Sv48, and Sv39) + * + * This recursive usage creates some duplicate mappings within + * unused portions of the virtual address space, but allows us to + * use only a single physical 4kB page table. + */ + +/** SBI base extension */ +#define SBI_BASE 0x10 +#define SBI_BASE_MVENDORID 0x04 + +/** Non-standard T-Head page table entry additional flags + * + * T-Head processors such as the C910 use the high bits of the PTE in + * a very non-standard way that is incompatible with the RISC-V + * specification. + * + * As per the "Memory Attribute Extension (XTheadMae)", bits 62 and 61 + * represent cacheability and "bufferability" (i.e. write-back + * cacheability) respectively. If we do not enable these bits, then + * the processor gets incredibly confused at the point that paging is + * enabled. The symptom is that cache lines will occasionally fail to + * fill, and so reads from any address may return unrelated data from + * a previously read cache line for a different address. + */ +#define THEAD_PTE_MAEE ( 0x60 << ( __riscv_xlen - 8 ) ) + +/** T-Head vendor ID */ +#define THEAD_MVENDORID 0x5b7 + +/** T-Head "sxstatus" CSR */ +#define THEAD_CSR_SXSTATUS 0x5c0 +#define THEAD_CSR_SXSTATUS_MAEE 0x00200000 /**< XTheadMae enabled */ + + .section ".prefix.enable_paging_64", "ax", @progbits +enable_paging_64: + /* Register usage: + * + * tp - return value (virtual address offset) + * a0 - page table base address + * a1 - currently attempted paging level + * a2 - enabled paging level + * a3 - PTE pointer + * a4 - PTE stride + * a5 - size of accessible physical address space + */ + progress " paging:" + + /* Calculate virtual address offset */ + LOADN t0, prefix_link + la t1, _prefix + sub tp, t1, t0 + + /* Zero PTE[0-511] */ + li t0, PTE_COUNT + mv a3, a0 +1: STOREN zero, (a3) + addi a3, a3, PTE_SIZE + addi t0, t0, -1 + bgtz t0, 1b + + /* Construct PTE[511] as next level page table pointer */ + srli t0, a0, PTE_PPN_SHIFT + ori t0, t0, PTE_V + STOREN t0, -PTE_SIZE(a3) + + /* Construct base page table entry for address zero */ + li t0, PTE_LEAF + STOREN t0, (a0) + + /* Check for broken T-Head paging extensions */ + mv a3, a0 + li a7, SBI_BASE + li a6, SBI_BASE_MVENDORID + ecall + bnez a0, 1f + li t0, THEAD_MVENDORID + bne a1, t0, 1f + progress "thead-" + csrr t0, THEAD_CSR_SXSTATUS + li t1, THEAD_CSR_SXSTATUS_MAEE + and t0, t0, t1 + beqz t0, 1f + progress "mae-" + LOADN t0, (a3) + li t1, THEAD_PTE_MAEE + or t0, t0, t1 + STOREN t0, (a3) +1: mv a0, a3 + + /* Calculate PTE[x] address for iPXE virtual address map */ + LOADN t0, prefix_link + srli t0, t0, VPN1_LSB + andi t0, t0, ( PTE_COUNT - 1 ) + slli t0, t0, PTE_SIZE_LOG2 + add a3, a0, t0 + + /* Calculate PTE stride for iPXE virtual address map + * + * PPN[1] LSB is PTE bit 19 in all paging modes, and so the + * stride is always ( 1 << 19 ) + */ + li a4, 1 + slli a4, a4, PTE_PPN1_LSB + + /* Construct PTE[x-1] for early UART, if applicable */ +#ifdef EARLY_UART_REG_BASE + li t0, ( EARLY_UART_REG_BASE & ~( ( 1 << VPN1_LSB ) - 1 ) ) + srli t0, t0, PTE_PPN_SHIFT + ori t0, t0, ( PTE_LEAF & ~PTE_X ) + STOREN t0, -PTE_SIZE(a3) +#endif + + /* Construct PTE[x-y] for iPXE virtual address map */ + la t0, _prefix + srli t0, t0, PTE_PPN_SHIFT + LOADN t1, (a0) + or t0, t0, t1 + la t2, _ebss + srli t2, t2, PTE_PPN_SHIFT +1: STOREN t0, (a3) + addi a3, a3, PTE_SIZE + add t0, t0, a4 + ble t0, t2, 1b + + /* Find highest supported paging level */ + li a1, SATP_MODE_SV57 +enable_paging_64_loop: + + /* Calculate PTE stride for identity map at this paging level + * + * a1 == 10 == Sv57: PPN[4] LSB is PTE bit 46 => stride := 1 << 46 + * a1 == 9 == Sv48: PPN[3] LSB is PTE bit 37 => stride := 1 << 37 + * a1 == 8 == Sv39: PPN[2] LSB is PTE bit 28 => stride := 1 << 28 + * + * and so we calculate stride a4 := ( 1 << ( 9 * a1 - 44 ) ) + */ + slli a4, a1, 3 + add a4, a4, a1 + addi a4, a4, -44 + li t0, 1 + sll a4, t0, a4 + + /* Calculate size of accessible physical address space + * + * The identity map comprises only the lower half of the PTEs, + * since virtual addresses for the higher half must have all + * high bits set, and so cannot form part of an identity map. + */ + slli a5, a4, ( PTE_PPN_SHIFT + ( PTE_COUNT_LOG2 - 1 ) ) + + /* Construct PTE[0-255] for identity map at this paging level */ + mv a3, a0 + li t0, ( PTE_COUNT / 2 ) + LOADN t1, (a0) +1: STOREN t1, (a3) + addi a3, a3, PTE_SIZE + add t1, t1, a4 + addi t0, t0, -1 + bgtz t0, 1b + + /* Attempt to enable paging, and read back active paging level */ + slli t0, a1, SATP_MODE_SHIFT + srli t1, a0, PAGE_SHIFT + or t0, t0, t1 + csrw satp, t0 + sfence.vma + csrr a2, satp + srli a2, a2, SATP_MODE_SHIFT + + /* Loop until we successfully enable paging, or run out of levels */ + beq a2, a1, 1f + csrw satp, zero + addi a1, a1, -1 + li t0, SATP_MODE_SV39 + bge a1, t0, enable_paging_64_loop + mv tp, zero + mv a5, zero +1: + /* Adjust return address to a virtual address */ + sub ra, ra, tp + + /* Return, with or without paging enabled */ + paging_mode_name a2 + mv a0, a5 + ret + .size enable_paging_64, . - enable_paging_64 + + /* Convert 64-bit physical address to virtual address */ + .macro phys_to_virt_64 rd, rs:vararg + .ifnb \rs + mv \rd, \rs + .endif + .endm + + /* Early UART base address when 64-bit paging is enabled + * + * When an early UART is in use, we choose to use the 2MB + * "megapage" immediately below iPXE itself to map the UART. + */ +#ifdef EARLY_UART_REG_BASE + .section ".rodata.early_uart_reg_base_64_virt", "a", @progbits + .balign 8 +early_uart_reg_base_64_virt: + .dword ( _base - ( 1 << VPN1_LSB ) + \ + ( EARLY_UART_REG_BASE & ( ( 1 << VPN1_LSB ) - 1 ) ) ) + .size early_uart_reg_base_64_virt, . - early_uart_reg_base_64_virt +#endif + +/***************************************************************************** + * + * Disable 64-bit paging + * + ***************************************************************************** + * + * This function may be called with either virtual or flat physical + * addressing. It does not require a valid stack pointer. + * + * Parameters: + * + * tp - Virtual address offset + * + * Returns: + * + * tp - Virtual address offset (zeroed) + * pc - Updated to a physical address + * + */ + + .section ".prefix.disable_paging_64", "ax", @progbits +disable_paging_64: + /* Register usage: + * + * tp - virtual address offset + */ + + /* Jump to physical address */ + la t0, 1f + bgez t0, 1f + add t0, t0, tp + jr t0 +1: + /* Disable paging */ + csrw satp, zero + sfence.vma + + /* Update return address to a physical address */ + bgez ra, 1f + add ra, ra, tp +1: + /* Return with paging disabled and virtual offset zeroed */ + mv tp, zero + ret + .size disable_paging_64, . - disable_paging_64 + +/***************************************************************************** + * + * Enable 32-bit paging + * + ***************************************************************************** + * + * Construct a 32-bit page table to map the whole of the 32-bit + * address space with a fixed offset selected to map iPXE itself at + * its link-time address (which must be 4MB-aligned). + * + * This function must be called with flat physical addressing. It + * does not require a valid stack pointer. + * + * Parameters: + * + * a0 - Page table to fill in (4kB, must be aligned to a 4kB boundary) + * + * Returns: + * + * a0 - Size of accessible physical address space (or zero for no limit) + * tp - Virtual address offset + * pc - Updated to a virtual address if paging enabled + * + * A 4kB 32-bit page table contains 1024 4-byte PTEs. We choose to + * use these to produce a circular map of the 32-bit address space + * using 4MB "megapages", with a fixed offset to align the virtual and + * link-time addresses. + * + * To handle the transition from physical to virtual addresses, we + * temporarily adjust the PTE covering the current program counter to + * be a direct physical map (so that the program counter remains valid + * at the moment when paging is enabled), then jump to a virtual + * address, then restore the temporarily modified PTE. + */ + + .equ enable_paging_32_xalign, 32 + + .section ".prefix.enable_paging_32", "ax", @progbits +enable_paging_32: + /* Register usage: + * + * tp - return value (virtual address offset) + * a0 - page table base address + * a1 - enabled paging level + * a2 - PTE pointer + * a3 - saved content of temporarily modified PTE + */ + progress " paging:" + + /* Calculate virtual address offset */ + LOADN t0, prefix_link + la t1, _prefix + sub tp, t1, t0 + + /* Construct PTEs for circular map */ + mv a2, a0 + li t0, PTE_COUNT + mv t1, tp + ori t1, t1, ( PTE_LEAF << PTE_PPN_SHIFT ) + li t2, ( 1 << ( PTE_PPN1_LSB + PTE_PPN_SHIFT ) ) +1: srli t3, t1, PTE_PPN_SHIFT + STOREN t3, (a2) + addi a2, a2, PTE_SIZE + add t1, t1, t2 + addi t0, t0, -1 + bgtz t0, 1b + + /* Temporarily modify PTE for transition code to be an identity map */ + la t0, enable_paging_32_xstart + srli t0, t0, VPN1_LSB + slli t1, t0, PTE_SIZE_LOG2 + add a2, a0, t1 + LOADN a3, (a2) + slli t0, t0, PTE_PPN1_LSB + ori t0, t0, PTE_LEAF + STOREN t0, (a2) + + /* Adjust PTE pointer to a virtual address */ + sub a2, a2, tp + + /* Attempt to enable paging, and read back active paging level */ + la t0, 1f + sub t0, t0, tp + li t1, ( SATP_MODE_SV32 << SATP_MODE_SHIFT ) + srli t2, a0, PAGE_SHIFT + or t1, t1, t2 + .balign enable_paging_32_xalign + /* Start of transition code */ +enable_paging_32_xstart: + csrw satp, t1 + sfence.vma + csrr a1, satp + beqz a1, 2f + jr t0 +1: /* Restore temporarily modified PTE */ + STOREN a3, (a2) + sfence.vma + /* End of transition code */ + .equ enable_paging_32_xlen, . - enable_paging_32_xstart +2: srli a1, a1, SATP_MODE_SHIFT + + /* Zero SATP and virtual address offset if paging is not enabled */ + bnez a1, 1f + csrw satp, zero + mv tp, zero +1: + /* Adjust return address to a virtual address */ + sub ra, ra, tp + + /* Return, with or without paging enabled */ + paging_mode_name a1 + mv a0, zero + ret + .size enable_paging_32, . - enable_paging_32 + + /* Ensure that transition code did not cross an alignment boundary */ + .section ".bss.enable_paging_32_xcheck", "aw", @nobits + .org . + enable_paging_32_xalign - enable_paging_32_xlen + + /* Convert 32-bit physical address to virtual address */ + .macro phys_to_virt_32 rd, rs:vararg + .ifnb \rs + sub \rd, \rs, tp + .else + sub \rd, \rd, tp + .endif + .endm + +/***************************************************************************** + * + * Disable 32-bit paging + * + ***************************************************************************** + * + * This function may be called with either virtual or flat physical + * addressing. It does not require a valid stack pointer. + * + * Parameters: + * + * tp - Virtual address offset + * + * Returns: + * + * tp - Virtual address offset (zeroed) + * pc - Updated to a physical address + * + */ + + .equ disable_paging_32_xalign, 16 + + .section ".prefix.disable_paging_32", "ax", @progbits +disable_paging_32: + /* Register usage: + * + * tp - virtual address offset + * a0 - page table address + * a1 - transition PTE pointer + * a2 - transition PTE content + */ + + /* Get page table address, and exit if paging is already disabled */ + csrr a0, satp + beqz a0, 99f + slli a0, a0, PAGE_SHIFT + sub a0, a0, tp + + /* Prepare for modifying transition PTE */ + la t0, disable_paging_32_xstart + add t0, t0, tp + srli t0, t0, VPN1_LSB + slli a1, t0, PTE_SIZE_LOG2 + add a1, a1, a0 + slli a2, t0, PTE_PPN1_LSB + ori a2, a2, PTE_LEAF + + /* Jump to physical address in transition PTE, and disable paging */ + la t0, 1f + add t0, t0, tp + .balign disable_paging_32_xalign + /* Start of transition code */ +disable_paging_32_xstart: + STOREN a2, (a1) + sfence.vma + jr t0 +1: csrw satp, zero + sfence.vma + /* End of transition code */ + .equ disable_paging_32_xlen, . - disable_paging_32_xstart + + /* Update return address to a physical address */ + add ra, ra, tp + +99: /* Return with paging disabled and virtual offset zeroed */ + mv tp, zero + ret + .size disable_paging_32, . - disable_paging_32 + + /* Ensure that transition code did not cross an alignment boundary */ + .section ".bss.disable_paging_32_xcheck", "aw", @nobits + .org . + disable_paging_32_xalign - disable_paging_32_xlen + +/***************************************************************************** + * + * Poison .bss section + * + ***************************************************************************** + * + * Fill the .bss section with an invalid non-zero value to expose bugs + * in early initialisation code that erroneously relies upon variables + * in .bss before the section has been zeroed. + * + * We use the value 0xeb55eb55eb55eb55 ("EBSS") since this is + * immediately recognisable as a value in a crash dump, and will + * trigger a page fault if dereferenced since the address is in a + * non-canonical form. + * + * Poisoning the .bss will overwrite the relocation records, and so + * can be done only as a debugging step on a system where relocation + * is known to be unnecessary (e.g. because paging is supported). + * + * This function does not require a valid stack pointer, but will + * destroy any existing stack contents if the stack happens to be + * placed within the original .bss section. + * + * Parameters: none + * + * Returns: none + * + */ + + .equ poison_bss_value_32, 0xeb55eb55 + .equ poison_bss_value_64, 0xeb55eb55eb55eb55 + .equ poison_bss_value, _C2 ( poison_bss_value_, __riscv_xlen ) + + .section ".prefix.poison_bss", "ax", @progbits +poison_bss: + /* Fill .bss section */ + la t0, _bss + la t1, _ebss + li t2, poison_bss_value +1: STOREN t2, (t0) + addi t0, t0, ( __riscv_xlen / 8 ) + blt t0, t1, 1b + ret + .size poison_bss, . - poison_bss + +/***************************************************************************** + * + * Install iPXE to a suitable runtime address + * + ***************************************************************************** + * + * Identify a suitable runtime address for iPXE, relocate there, and + * set up for running normal C code. + * + * A valid temporary stack pointer is required. A 4kB space for a + * temporary page table may be provided, and must be provided if the + * iPXE image is running from read-only memory. + * + * Note that this function does not preserve the callee-save registers. + * + * Parameters: + * + * a0 - Boot hart ID + * a1 - Device tree physical address + * a2 - Optional temporary page table space (4kB, aligned to a 4kB boundary) + * sp - Valid temporary stack pointer + * + * Returns: + * + * pc - Updated to be within the relocated iPXE + * sp - Top of internal stack + * tp - Virtual address offset + * + */ + + .section ".prefix.install", "ax", @progbits + .globl install +install: + /* Register usage: + * + * s0 - boot hart ID + * s1 - device tree physical address + * s2 - saved return address + * s3 - relocation records physical address + * s4 - maximum accessible physical address + * s5 - relocation physical address + * s6 - relocation offset + * tp - virtual address offset + */ + mv tp, zero + progress "\r\nSBI->iPXE hart:" + print_hex_reg a0 + progress " temp:" + print_hex_reg a2 + progress " fdt:" + print_hex_reg a1 + progress "\r\nSBI->iPXE phys:" + print_hex_addr _prefix + progress " virt:" + print_hex_data prefix_virt + mv s0, a0 + mv s1, a1 + mv s2, ra + la s3, _edata + + /* Poison .bss if configured to do so */ +#if POISON_BSS + call poison_bss +#endif + + /* Attempt to enable paging, if we have temporary page table space */ + mv a0, a2 + beqz a2, 1f + call enable_paging +1: addi s4, a0, -1 + + /* Apply relocations, if still needed after enabling paging */ + mv a0, s3 + call apply_relocs + + /* Find a suitable address for relocation (using temporary stack) */ + phys_to_virt a0, s1 + mv a1, s4 + phys_to_virt sp + call fdtmem_relocate + mv s5, a0 + progress "SBI->iPXE dest:" + print_hex_reg a0 + + /* Disable paging */ + call disable_paging + + /* Determine relocation offset */ + la s6, _prefix + sub s6, s5, s6 + + /* Copy iPXE image to new location and zero .bss */ + mv t0, s5 + la t1, _prefix + la t2, _edata +1: LOADN t3, (t1) + STOREN t3, (t0) + addi t0, t0, ( __riscv_xlen / 8 ) + addi t1, t1, ( __riscv_xlen / 8 ) + blt t1, t2, 1b + la t1, _ebss + add t1, t1, s6 +2: STOREN zero, (t0) + addi t0, t0, ( __riscv_xlen / 8 ) + blt t0, t1, 2b + + /* Jump to relocated copy */ + la t0, 1f + add t0, t0, s6 + jr t0 +1: + /* Attempt to re-enable paging */ + la a0, page_table + call enable_paging + + /* Reapply relocations, if still needed after enabling paging */ + phys_to_virt a0, s3 + call apply_relocs + + /* Load stack pointer */ + la sp, _estack + + /* Store boot hart */ + STOREN s0, boot_hart, t0 + + /* Copy and register system device tree */ + phys_to_virt a0, s1 + mv a1, s4 + call fdtmem_register + + /* Return to a virtual address in the relocated copy */ + add ra, s2, s6 + sub ra, ra, tp + progress "\r\n" + ret + .size install, . - install + +/***************************************************************************** + * + * Reset (or lock up) system + * + ***************************************************************************** + * + * Reset via system via SBI, as a means of exiting from a prefix that + * has no other defined exit path. If the reset fails, lock up the + * system since there is nothing else that can sensibly be done. + * + * This function does not require a valid stack pointer. + * + * Parameters: none + * + * Returns: n/a (does not return) + * + */ + +/* SBI system reset extension */ +#define SBI_SRST ( ( 'S' << 24 ) | ( 'R' << 16 ) | ( 'S' << 8 ) | 'T' ) +#define SBI_SRST_SYSTEM_RESET 0x00 +#define SBI_RESET_COLD 0x00000001 + +/* SBI legacy shutdown */ +#define SBI_LEGACY_SHUTDOWN 0x08 + + .section ".prefix.reset_system", "ax", @progbits + .globl reset_system +reset_system: + /* Register usage: irrelevant (does not return) */ + progress "\r\niPXE->SBI reset\r\n" + + /* Attempt reset */ + li a7, SBI_SRST + li a6, SBI_SRST_SYSTEM_RESET + li a0, SBI_RESET_COLD + mv a1, zero + ecall + progress "(reset failed)\r\n" + + /* Attempt legacy shutdown */ + li a7, SBI_LEGACY_SHUTDOWN + ecall + progress "(legacy shutdown failed)\r\n" + + /* If reset failed, lock the system */ +1: wfi + j 1b + .size reset_system, . - reset_system + +/***************************************************************************** + * + * File split information for the compressor + * + ***************************************************************************** + */ + +/* ELF machine type */ +#define EM_RISCV 243 + + .section ".zinfo", "a", @progbits + .org 0 + /* Copy initialised-data portion of image */ + .ascii "COPY" + .word 0 + .word _filesz + .word 1 + /* Notify compressor of link-time base address */ + .ascii "BASE" + .word 0 + .dword _base + /* Construct compressed relocation records */ + .ascii "ZREL" + .word _reloc_offset + .word _reloc_filesz + .word EM_RISCV diff --git a/src/arch/riscv/prefix/lkrnprefix.S b/src/arch/riscv/prefix/lkrnprefix.S new file mode 100644 index 000000000..9493d016c --- /dev/null +++ b/src/arch/riscv/prefix/lkrnprefix.S @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * Linux kernel prefix + * + */ + + .section ".note.GNU-stack", "", @progbits + .text + + /* Layout of kernel header */ + .struct 0 +hdr_code0: .space 4 +hdr_code1: .space 4 +hdr_text_offset: .space 8 +hdr_image_size: .space 8 +hdr_flags: .space 8 +hdr_version: .space 4 +hdr_res1: .space 4 +hdr_res2: .space 8 +hdr_magic: .space 8 +hdr_magic2: .space 4 +hdr_res3: .space 4 +hdr_end: + .org 64 + .previous + +/* Header version */ +#define HDR_VERSION( major, minor ) ( ( (major) << 16 ) | (minor) ) +#define HDR_VERSION_0_2 HDR_VERSION ( 0, 2 ) + +/* Header flags */ +#define HDR_FL_BIG_ENDIAN 0x00000001 + +/* Magic numbers */ +#define HDR_MAGIC "RISCV\0\0\0" +#define HDR_MAGIC2 "RSC\x05" + + /* + * Linux kernel header + */ + .section ".prefix", "ax", @progbits + + /* Executable code / MZ header (for EFI-compatible binaries) */ + .org hdr_code0 + j _lkrn_start + + /* Image load offset + * + * Must be set to the size of a single "megapage" (2MB for + * 64-bit, 4MB for 32-bit). + */ + .org hdr_text_offset + .dword _max_align + + /* Image size (including uninitialised-data potions) */ + .org hdr_image_size + .dword _memsz + + /* Flags */ + .org hdr_flags + .dword 0 + + /* Version */ + .org hdr_version + .word HDR_VERSION_0_2 + + /* Magic numbers */ + .org hdr_magic + .ascii HDR_MAGIC + .org hdr_magic2 + .ascii HDR_MAGIC2 + + .org hdr_end + + /* + * Linux kernel entry point + */ + .globl _lkrn_start +_lkrn_start: + /* Identify temporary page table and stack space + * + * Linux expects to be placed at the image load offset from + * the start of RAM. Assume that our loaded image is + * therefore already writable, and that we can therefore use + * the page table and stack within our (not yet zeroed) .bss + * section. + */ + la a2, page_table + la sp, _estack + + /* Install iPXE */ + call install + + /* Call main program */ + call main + + /* We have no return path, since the Linux kernel does not + * define that a valid return address exists. + * + * Attempt a system reset, since there is nothing else we can + * viably do at this point. + */ + j reset_system + .size _lkrn_start, . - _lkrn_start diff --git a/src/arch/riscv/prefix/sbiprefix.S b/src/arch/riscv/prefix/sbiprefix.S new file mode 100644 index 000000000..da3202aa2 --- /dev/null +++ b/src/arch/riscv/prefix/sbiprefix.S @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * SBI position-independent executable prefix + * + */ + + .section ".note.GNU-stack", "", @progbits + .text + +/* Page size */ +#define PAGE_SIZE 4096 + + /* + * SBI entry point + */ + .section ".prefix", "ax", @progbits + .org 0 + .globl _sbi_start +_sbi_start: + /* Identify temporary page table and stack space + * + * Assume that there is sufficient writable memory (~8kB) + * directly below the device tree. + */ + li t0, ~( PAGE_SIZE - 1 ) + and sp, a1, t0 + li t0, PAGE_SIZE + sub sp, sp, t0 + mv a2, sp + + /* Install iPXE */ + call install + + /* Call main program */ + call main + + /* We have no return path, since the M-mode SBI implementation + * will have jumped to us by setting our start address in MEPC + * and issuing an MRET instruction. + * + * Attempt a system reset, since there is nothing else we can + * viably do at this point. + */ + j reset_system + .size _sbi_start, . - _sbi_start diff --git a/src/arch/riscv/scripts/sbi.lds b/src/arch/riscv/scripts/sbi.lds new file mode 100644 index 000000000..04cabcf85 --- /dev/null +++ b/src/arch/riscv/scripts/sbi.lds @@ -0,0 +1,133 @@ +/* + * Linker script for RISC-V SBI images + * + */ + +SECTIONS { + + /* Weak symbols that need zero values if not otherwise defined */ + saved_pos = .; + .weak 0x0 : { + _weak = .; + *(.weak) + *(.weak.*) + _eweak = .; + } + _assert = ASSERT ( ( _weak == _eweak ), ".weak is non-zero length" ); + _assert = ASSERT ( ( . == saved_pos ), ".weak altered current position" ); + + /* Prefix code */ + .prefix : { + _prefix = .; + *(.prefix) + *(.prefix.*) + _eprefix = .; + } + + /* Program code */ + .text : { + _text = .; + *(.text) + *(.text.*) + _etext = .; + } + + /* Align to page size to allow linker to generate W^X segments */ + . = ALIGN ( 4096 ); + + /* Read-only data */ + .rodata : { + _rodata = .; + *(.srodata) + *(.srodata.*) + *(.rodata) + *(.rodata.*) + _erodata = .; + } + + /* Writable data */ + .data : { + _data = .; + *(.sdata) + *(.sdata.*) + *(.data) + *(.data.*) + KEEP(*(SORT(.tbl.*))) /* Various tables. See include/tables.h */ + KEEP(*(.provided)) + KEEP(*(.provided.*)) + *(.got) + *(.got.plt) + /* Ensure compressed relocations end up aligned */ + . = ALIGN ( 16 ); + _edata = .; + } + + /* Uninitialised and discardable data */ + OVERLAY : { + + /* Runtime relocations (discarded after use) */ + .rela.dyn { + *(.rela) + *(.rela.dyn) + } + + /* Compressor information block */ + .zinfo { + _zinfo = .; + KEEP(*(.zinfo)) + KEEP(*(.zinfo.*)) + _ezinfo = .; + } + + /* Uninitialised data */ + .bss { + _bss = .; + *(.sbss) + *(.sbss.*) + *(.bss) + *(.bss.*) + *(COMMON) + *(.stack) + *(.stack.*) + /* Align to allow for easy zeroing by prefix code */ + . = ALIGN ( 16 ); + _ebss = .; + } + } + + /* End virtual address */ + _end = .; + + /* Base virtual address */ + _base = ABSOLUTE ( _prefix ); + + /* Relocations */ + _reloc_offset = ( LOADADDR ( .rela.dyn ) - LOADADDR ( .prefix ) ); + _reloc_filesz = SIZEOF ( .rela.dyn ); + + /* Length of initialised data */ + _filesz = ( ABSOLUTE ( _edata ) - ABSOLUTE ( _prefix ) ); + + /* Length of in-memory image */ + _memsz = ( ABSOLUTE ( _end ) - ABSOLUTE ( _prefix ) ); + + /* Unwanted sections */ + /DISCARD/ : { + *(.comment) + *(.comment.*) + *(.note) + *(.note.*) + *(.eh_frame) + *(.eh_frame.*) + *(.dynamic) + *(.dynsym) + *(.dynstr) + *(.hash) + *(.gnu.hash) + *(.einfo) + *(.einfo.*) + *(.discard) + *(.discard.*) + *(.pci_devlist.*) + } +} diff --git a/src/arch/riscv32/Makefile b/src/arch/riscv32/Makefile new file mode 100644 index 000000000..0d8b7ef50 --- /dev/null +++ b/src/arch/riscv32/Makefile @@ -0,0 +1,24 @@ +# Specify compressor +# +ZBIN = $(ZBIN32) + +# RISCV32-specific directories containing source files +# +SRCDIRS += arch/riscv32/core +SRCDIRS += arch/riscv32/libgcc + +# RISCV32-specific flags +# +CFLAGS += -march=rv32gc -mabi=ilp32d +ASFLAGS += -march=rv32gc -mabi=ilp32d +LDFLAGS += -m elf32lriscv + +# Include common RISCV Makefile +# +MAKEDEPS += arch/riscv/Makefile +include arch/riscv/Makefile + +# Include platform-specific Makefile +# +MAKEDEPS += arch/riscv32/Makefile.$(PLATFORM) +include arch/riscv32/Makefile.$(PLATFORM) diff --git a/src/arch/riscv32/Makefile.efi b/src/arch/riscv32/Makefile.efi new file mode 100644 index 000000000..2cc1a930f --- /dev/null +++ b/src/arch/riscv32/Makefile.efi @@ -0,0 +1,10 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# Specify EFI image builder +# +ELF2EFI = $(ELF2EFI32) + +# Include generic EFI Makefile +# +MAKEDEPS += arch/riscv/Makefile.efi +include arch/riscv/Makefile.efi diff --git a/src/arch/riscv32/Makefile.linux b/src/arch/riscv32/Makefile.linux new file mode 100644 index 000000000..86ac6b8d5 --- /dev/null +++ b/src/arch/riscv32/Makefile.linux @@ -0,0 +1,14 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# Starting virtual address +# +LDFLAGS += -Ttext=0x10000 + +# Compiler flags for building host API wrapper +# +LINUX_CFLAGS += -march=rv32gc -mabi=ilp32d + +# Include generic Linux Makefile +# +MAKEDEPS += arch/riscv/Makefile.linux +include arch/riscv/Makefile.linux diff --git a/src/arch/riscv32/Makefile.sbi b/src/arch/riscv32/Makefile.sbi new file mode 100644 index 000000000..5841dd4b5 --- /dev/null +++ b/src/arch/riscv32/Makefile.sbi @@ -0,0 +1,14 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# Set base virtual address to 0xeb000000 +# +# This is aligned to a 4MB boundary and so allows 4MB megapages to be +# used to map the iPXE binary. The address pattern is also easily +# recognisable if leaked to unexpected contexts. +# +LDFLAGS += --section-start=.prefix=0xeb000000 + +# Include generic SBI Makefile +# +MAKEDEPS += arch/riscv/Makefile.sbi +include arch/riscv/Makefile.sbi diff --git a/src/arch/riscv32/core/riscv32_byteswap.S b/src/arch/riscv32/core/riscv32_byteswap.S new file mode 100644 index 000000000..571431f35 --- /dev/null +++ b/src/arch/riscv32/core/riscv32_byteswap.S @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * Byte swapping + * + */ + + .section ".note.GNU-stack", "", @progbits + .text + + .section ".text.riscv_swap", "ax", @progbits +riscv_swap: + .globl riscv_swap_word + .globl riscv_swap_half + .globl riscv_swap_byte +riscv_swap_word: + /* Swap all bytes in a0 */ + mv t0, ra + jal riscv_swap_half + mv ra, t0 + /* Swap words a0 and a1 */ + mv t1, a0 + mv a0, a1 + mv a1, t1 +riscv_swap_half: + /* Swap half-words within a0 */ + slli t2, a0, 16 + srli a0, a0, 16 + or a0, a0, t2 +riscv_swap_byte: + /* Swap bytes within each half-word of a0 */ + li t3, 0xff00ff00 + slli t4, a0, 8 + and a0, a0, t3 + and t4, t4, t3 + srli a0, a0, 8 + or a0, a0, t4 + ret + .size riscv_swap, . - riscv_swap diff --git a/src/arch/riscv32/include/ipxe/efi/dhcparch.h b/src/arch/riscv32/include/ipxe/efi/dhcparch.h new file mode 100644 index 000000000..b74df1925 --- /dev/null +++ b/src/arch/riscv32/include/ipxe/efi/dhcparch.h @@ -0,0 +1,20 @@ +#ifndef _IPXE_EFI_DHCPARCH_H +#define _IPXE_EFI_DHCPARCH_H + +/** @file + * + * DHCP client architecture definitions + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <ipxe/dhcp.h> + +/** DHCP client architecture */ +#define DHCP_ARCH_CLIENT_ARCHITECTURE DHCP_CLIENT_ARCHITECTURE_RISCV32 + +/** DHCP client network device interface */ +#define DHCP_ARCH_CLIENT_NDI 1 /* UNDI */ , 3, 10 /* v3.10 */ + +#endif /* _IPXE_EFI_DHCPARCH_H */ diff --git a/src/arch/riscv32/include/ipxe/sbi/dhcparch.h b/src/arch/riscv32/include/ipxe/sbi/dhcparch.h new file mode 100644 index 000000000..713d4cf5d --- /dev/null +++ b/src/arch/riscv32/include/ipxe/sbi/dhcparch.h @@ -0,0 +1,20 @@ +#ifndef _IPXE_SBI_DHCPARCH_H +#define _IPXE_SBI_DHCPARCH_H + +/** @file + * + * DHCP client architecture definitions + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <ipxe/dhcp.h> + +/** DHCP client architecture */ +#define DHCP_ARCH_CLIENT_ARCHITECTURE DHCP_CLIENT_ARCHITECTURE_RISCV32 + +/** DHCP client network device interface */ +#define DHCP_ARCH_CLIENT_NDI 1 /* UNDI */ , 3, 10 /* v3.10 */ + +#endif /* _IPXE_SBI_DHCPARCH_H */ diff --git a/src/arch/riscv32/include/limits.h b/src/arch/riscv32/include/limits.h new file mode 100644 index 000000000..bb48b75ab --- /dev/null +++ b/src/arch/riscv32/include/limits.h @@ -0,0 +1,61 @@ +#ifndef LIMITS_H +#define LIMITS_H 1 + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/* Number of bits in a `char' */ +#define CHAR_BIT 8 + +/* Minimum and maximum values a `signed char' can hold */ +#define SCHAR_MIN (-128) +#define SCHAR_MAX 127 + +/* Maximum value an `unsigned char' can hold. (Minimum is 0.) */ +#define UCHAR_MAX 255 + +/* Minimum and maximum values a `char' can hold */ +#define CHAR_MIN SCHAR_MIN +#define CHAR_MAX SCHAR_MAX + +/* Minimum and maximum values a `signed short int' can hold */ +#define SHRT_MIN (-32768) +#define SHRT_MAX 32767 + +/* Maximum value an `unsigned short' can hold. (Minimum is 0.) */ +#define USHRT_MAX 65535 + + +/* Minimum and maximum values a `signed int' can hold */ +#define INT_MIN (-INT_MAX - 1) +#define INT_MAX 2147483647 + +/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */ +#define UINT_MAX 4294967295U + + +/* Minimum and maximum values a `signed int' can hold */ +#define INT_MAX 2147483647 +#define INT_MIN (-INT_MAX - 1) + + +/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */ +#define UINT_MAX 4294967295U + + +/* Minimum and maximum values a `signed long' can hold */ +#define LONG_MAX 2147483647 +#define LONG_MIN (-LONG_MAX - 1L) + +/* Maximum value an `unsigned long' can hold. (Minimum is 0.) */ +#define ULONG_MAX 4294967295UL + +/* Minimum and maximum values a `signed long long' can hold */ +#define LLONG_MAX 9223372036854775807LL +#define LLONG_MIN (-LONG_MAX - 1LL) + + +/* Maximum value an `unsigned long long' can hold. (Minimum is 0.) */ +#define ULLONG_MAX 18446744073709551615ULL + + +#endif /* LIMITS_H */ diff --git a/src/arch/riscv32/libgcc/llshift.S b/src/arch/riscv32/libgcc/llshift.S new file mode 100644 index 000000000..8ad6a6daa --- /dev/null +++ b/src/arch/riscv32/libgcc/llshift.S @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * Long shifts + * + */ + + .section ".note.GNU-stack", "", @progbits + .text + +/** + * Shift left + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + .section ".text.__ashldi3", "ax", @progbits + .globl __ashldi3 +__ashldi3: + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a1, a0 + mv a0, zero +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + srl t2, a0, t1 + sll a0, a0, a2 + sll a1, a1, a2 + or a1, a1, t2 +2: ret + .size __ashldi3, . - __ashldi3 + +/** + * Logical shift right + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + .section ".text.__lshrdi3", "ax", @progbits + .globl __lshrdi3 +__lshrdi3: + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a0, a1 + mv a1, zero +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + sll t2, a1, t1 + srl a1, a1, a2 + srl a0, a0, a2 + or a0, a0, t2 +2: ret + .size __lshrdi3, . - __lshrdi3 + +/** + * Arithmetic shift right + * + * @v a1:a0 Value to shift + * @v a2 Shift amount + * @ret a1:a0 Shifted value + */ + .section ".text.__ashrdi3", "ax", @progbits + .globl __ashrdi3 +__ashrdi3: + /* Perform shift by 32 bits, if applicable */ + li t0, 32 + sub t1, t0, a2 + bgtz t1, 1f + mv a0, a1 + srai a1, a1, 16 + srai a1, a1, 16 +1: /* Perform shift by modulo-32 bits, if applicable */ + andi a2, a2, 0x1f + beqz a2, 2f + sll t2, a1, t1 + sra a1, a1, a2 + srl a0, a0, a2 + or a0, a0, t2 +2: ret + .size __ashrdi3, . - __ashrdi3 diff --git a/src/arch/riscv64/Makefile b/src/arch/riscv64/Makefile new file mode 100644 index 000000000..017fbacef --- /dev/null +++ b/src/arch/riscv64/Makefile @@ -0,0 +1,23 @@ +# Specify compressor +# +ZBIN = $(ZBIN64) + +# RISCV64-specific directories containing source files +# +SRCDIRS += arch/riscv64/core + +# RISCV64-specific flags +# +CFLAGS += -march=rv64gc -mabi=lp64d +ASFLAGS += -march=rv64gc -mabi=lp64d +LDFLAGS += -m elf64lriscv + +# Include common RISCV Makefile +# +MAKEDEPS += arch/riscv/Makefile +include arch/riscv/Makefile + +# Include platform-specific Makefile +# +MAKEDEPS += arch/riscv64/Makefile.$(PLATFORM) +include arch/riscv64/Makefile.$(PLATFORM) diff --git a/src/arch/riscv64/Makefile.efi b/src/arch/riscv64/Makefile.efi new file mode 100644 index 000000000..3948ca15c --- /dev/null +++ b/src/arch/riscv64/Makefile.efi @@ -0,0 +1,10 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# Specify EFI image builder +# +ELF2EFI = $(ELF2EFI64) + +# Include generic EFI Makefile +# +MAKEDEPS += arch/riscv/Makefile.efi +include arch/riscv/Makefile.efi diff --git a/src/arch/riscv64/Makefile.linux b/src/arch/riscv64/Makefile.linux new file mode 100644 index 000000000..5ceafed8c --- /dev/null +++ b/src/arch/riscv64/Makefile.linux @@ -0,0 +1,10 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# Starting virtual address +# +LDFLAGS += -Ttext=0x10000 + +# Include generic Linux Makefile +# +MAKEDEPS += arch/riscv/Makefile.linux +include arch/riscv/Makefile.linux diff --git a/src/arch/riscv64/Makefile.sbi b/src/arch/riscv64/Makefile.sbi new file mode 100644 index 000000000..0f7e1c373 --- /dev/null +++ b/src/arch/riscv64/Makefile.sbi @@ -0,0 +1,14 @@ +# -*- makefile -*- : Force emacs to use Makefile mode + +# Set base virtual address to 0xffffffffeb000000 +# +# This is aligned to a 2MB boundary and so allows 2MB megapages to be +# used to map the iPXE binary. The address pattern is also easily +# recognisable if leaked to unexpected contexts. +# +LDFLAGS += --section-start=.prefix=0xffffffffeb000000 + +# Include generic SBI Makefile +# +MAKEDEPS += arch/riscv/Makefile.sbi +include arch/riscv/Makefile.sbi diff --git a/src/arch/riscv64/core/riscv64_byteswap.S b/src/arch/riscv64/core/riscv64_byteswap.S new file mode 100644 index 000000000..ec2b0b9dd --- /dev/null +++ b/src/arch/riscv64/core/riscv64_byteswap.S @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2024 Michael Brown <mbrown@fensystems.co.uk>. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or any later version. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + * + * You can also choose to distribute this program under the terms of + * the Unmodified Binary Distribution Licence (as given in the file + * COPYING.UBDL), provided that you have satisfied its requirements. + */ + + FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) + +/** @file + * + * Byte swapping + * + */ + + .section ".note.GNU-stack", "", @progbits + .text + + .section ".text.riscv_swap", "ax", @progbits +riscv_swap: + .globl riscv_swap_word + .globl riscv_swap_half + .globl riscv_swap_byte +riscv_swap_word: + /* Swap low and high words of a0 */ + slli t0, a0, 32 + srli a0, a0, 32 + or a0, a0, t0 +riscv_swap_half: + /* Swap half-words within each word of a0 */ + ld t1, mask16 + slli t2, a0, 16 + and a0, a0, t1 + and t2, t2, t1 + srli a0, a0, 16 + or a0, a0, t2 +riscv_swap_byte: + /* Swap bytes within each half-word of a0 */ + ld t3, mask8 + slli t4, a0, 8 + and a0, a0, t3 + and t4, t4, t3 + srli a0, a0, 8 + or a0, a0, t4 + ret +mask16: .dword 0xffff0000ffff0000 +mask8: .dword 0xff00ff00ff00ff00 + .size riscv_swap, . - riscv_swap diff --git a/src/arch/riscv64/include/ipxe/efi/dhcparch.h b/src/arch/riscv64/include/ipxe/efi/dhcparch.h new file mode 100644 index 000000000..33bca044e --- /dev/null +++ b/src/arch/riscv64/include/ipxe/efi/dhcparch.h @@ -0,0 +1,20 @@ +#ifndef _IPXE_EFI_DHCPARCH_H +#define _IPXE_EFI_DHCPARCH_H + +/** @file + * + * DHCP client architecture definitions + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <ipxe/dhcp.h> + +/** DHCP client architecture */ +#define DHCP_ARCH_CLIENT_ARCHITECTURE DHCP_CLIENT_ARCHITECTURE_RISCV64 + +/** DHCP client network device interface */ +#define DHCP_ARCH_CLIENT_NDI 1 /* UNDI */ , 3, 10 /* v3.10 */ + +#endif /* _IPXE_EFI_DHCPARCH_H */ diff --git a/src/arch/riscv64/include/ipxe/sbi/dhcparch.h b/src/arch/riscv64/include/ipxe/sbi/dhcparch.h new file mode 100644 index 000000000..e172f064f --- /dev/null +++ b/src/arch/riscv64/include/ipxe/sbi/dhcparch.h @@ -0,0 +1,20 @@ +#ifndef _IPXE_SBI_DHCPARCH_H +#define _IPXE_SBI_DHCPARCH_H + +/** @file + * + * DHCP client architecture definitions + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <ipxe/dhcp.h> + +/** DHCP client architecture */ +#define DHCP_ARCH_CLIENT_ARCHITECTURE DHCP_CLIENT_ARCHITECTURE_RISCV64 + +/** DHCP client network device interface */ +#define DHCP_ARCH_CLIENT_NDI 1 /* UNDI */ , 3, 10 /* v3.10 */ + +#endif /* _IPXE_SBI_DHCPARCH_H */ diff --git a/src/arch/riscv64/include/limits.h b/src/arch/riscv64/include/limits.h new file mode 100644 index 000000000..a1374a17f --- /dev/null +++ b/src/arch/riscv64/include/limits.h @@ -0,0 +1,61 @@ +#ifndef LIMITS_H +#define LIMITS_H 1 + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +/* Number of bits in a `char' */ +#define CHAR_BIT 8 + +/* Minimum and maximum values a `signed char' can hold */ +#define SCHAR_MIN (-128) +#define SCHAR_MAX 127 + +/* Maximum value an `unsigned char' can hold. (Minimum is 0.) */ +#define UCHAR_MAX 255 + +/* Minimum and maximum values a `char' can hold */ +#define CHAR_MIN SCHAR_MIN +#define CHAR_MAX SCHAR_MAX + +/* Minimum and maximum values a `signed short int' can hold */ +#define SHRT_MIN (-32768) +#define SHRT_MAX 32767 + +/* Maximum value an `unsigned short' can hold. (Minimum is 0.) */ +#define USHRT_MAX 65535 + + +/* Minimum and maximum values a `signed int' can hold */ +#define INT_MIN (-INT_MAX - 1) +#define INT_MAX 2147483647 + +/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */ +#define UINT_MAX 4294967295U + + +/* Minimum and maximum values a `signed int' can hold */ +#define INT_MAX 2147483647 +#define INT_MIN (-INT_MAX - 1) + + +/* Maximum value an `unsigned int' can hold. (Minimum is 0.) */ +#define UINT_MAX 4294967295U + + +/* Minimum and maximum values a `signed long' can hold */ +#define LONG_MAX 9223372036854775807L +#define LONG_MIN (-LONG_MAX - 1L) + +/* Maximum value an `unsigned long' can hold. (Minimum is 0.) */ +#define ULONG_MAX 18446744073709551615UL + +/* Minimum and maximum values a `signed long long' can hold */ +#define LLONG_MAX 9223372036854775807LL +#define LLONG_MIN (-LONG_MAX - 1LL) + + +/* Maximum value an `unsigned long long' can hold. (Minimum is 0.) */ +#define ULLONG_MAX 18446744073709551615ULL + + +#endif /* LIMITS_H */ diff --git a/src/arch/x86/Makefile b/src/arch/x86/Makefile index ef801365e..4a4d8ee91 100644 --- a/src/arch/x86/Makefile +++ b/src/arch/x86/Makefile @@ -3,9 +3,9 @@ ASM_TCHAR := @ ASM_TCHAR_OPS := @ -# Include common x86 headers +# Include x86-specific headers # -INCDIRS += arch/x86/include +INCDIRS := arch/$(ARCH)/include arch/x86/include $(INCDIRS) # x86-specific directories containing source files # diff --git a/src/arch/x86/core/cpuid.c b/src/arch/x86/core/cpuid.c index 1a7c93e83..0461b846e 100644 --- a/src/arch/x86/core/cpuid.c +++ b/src/arch/x86/core/cpuid.c @@ -22,6 +22,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <string.h> #include <errno.h> @@ -84,8 +85,8 @@ int cpuid_supported ( uint32_t function ) { return rc; /* Find highest supported function number within this family */ - cpuid ( ( function & CPUID_EXTENDED ), 0, &max_function, &discard_b, - &discard_c, &discard_d ); + cpuid ( ( function & ( CPUID_EXTENDED | CPUID_HYPERVISOR ) ), 0, + &max_function, &discard_b, &discard_c, &discard_d ); /* Fail if maximum function number is meaningless (e.g. if we * are attempting to call an extended function on a CPU which diff --git a/src/arch/x86/core/cpuid_settings.c b/src/arch/x86/core/cpuid_settings.c index 0b67ee91d..ef0164069 100644 --- a/src/arch/x86/core/cpuid_settings.c +++ b/src/arch/x86/core/cpuid_settings.c @@ -22,6 +22,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <string.h> #include <errno.h> @@ -38,7 +39,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * * Bit 31 Extended function * Bits 30-24 (bit 22 = 1) Subfunction number - * (bit 22 = 0) Number of consecutive functions to call, minus one + * Bit 30 (bit 22 = 0) Hypervisor function + * Bits 29-24 (bit 22 = 0) Number of consecutive functions to call, minus one * Bit 23 Return result as little-endian (used for strings) * Bit 22 Interpret bits 30-24 as a subfunction number * Bits 21-18 Unused @@ -98,7 +100,7 @@ enum cpuid_flags { * @v tag Setting tag * @ret function Starting function number */ -#define CPUID_FUNCTION( tag ) ( (tag) & 0x800000ffUL ) +#define CPUID_FUNCTION( tag ) ( (tag) & 0xc00000ffUL ) /** * Extract subfunction number from CPUID setting tag @@ -109,6 +111,14 @@ enum cpuid_flags { #define CPUID_SUBFUNCTION( tag ) ( ( (tag) >> 24 ) & 0x7f ) /** + * Extract number of consecutive functions from CPUID setting tag + * + * @v tag Setting tag + * @ret num_functions Number of consecutive functions + */ +#define CPUID_NUM_FUNCTIONS( tag ) ( ( ( (tag) >> 24 ) & 0x3f ) + 1 ) + +/** * Extract register array from CPUID setting tag * * @v tag Setting tag @@ -165,12 +175,13 @@ static int cpuid_settings_fetch ( struct settings *settings, /* Call each function in turn */ function = CPUID_FUNCTION ( setting->tag ); - subfunction = CPUID_SUBFUNCTION ( setting->tag ); if ( setting->tag & CPUID_USE_SUBFUNCTION ) { + function &= ~CPUID_HYPERVISOR; + subfunction = CPUID_SUBFUNCTION ( setting->tag ); num_functions = 1; } else { - num_functions = ( subfunction + 1 ); subfunction = 0; + num_functions = CPUID_NUM_FUNCTIONS ( setting->tag ); } for ( ; num_functions-- ; function++ ) { @@ -240,6 +251,7 @@ static void cpuid_settings_init ( void ) { /** CPUID settings initialiser */ struct init_fn cpuid_settings_init_fn __init_fn ( INIT_NORMAL ) = { + .name = "cpuid", .initialise = cpuid_settings_init, }; diff --git a/src/arch/x86/core/debugcon.c b/src/arch/x86/core/debugcon.c index 60de61f55..0e3a5dfc7 100644 --- a/src/arch/x86/core/debugcon.c +++ b/src/arch/x86/core/debugcon.c @@ -86,5 +86,6 @@ static void debugcon_init ( void ) { * Debug port console initialisation function */ struct init_fn debugcon_init_fn __init_fn ( INIT_EARLY ) = { + .name = "debugcon", .initialise = debugcon_init, }; diff --git a/src/arch/x86/core/gdbmach.c b/src/arch/x86/core/gdbmach.c index af6abfedd..d4d187e35 100644 --- a/src/arch/x86/core/gdbmach.c +++ b/src/arch/x86/core/gdbmach.c @@ -31,7 +31,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <ipxe/uaccess.h> #include <ipxe/gdbstub.h> #include <librm.h> -#include <gdbmach.h> /** @file * diff --git a/src/arch/x86/core/pci_autoboot.c b/src/arch/x86/core/pci_autoboot.c index 337598091..243e45026 100644 --- a/src/arch/x86/core/pci_autoboot.c +++ b/src/arch/x86/core/pci_autoboot.c @@ -44,5 +44,6 @@ static void pci_autoboot_init ( void ) { /** PCI autoboot device initialisation function */ struct init_fn pci_autoboot_init_fn __init_fn ( INIT_NORMAL ) = { + .name = "autoboot", .initialise = pci_autoboot_init, }; diff --git a/src/arch/x86/core/pcidirect.c b/src/arch/x86/core/pcidirect.c index f4659a1ac..887b78a0b 100644 --- a/src/arch/x86/core/pcidirect.c +++ b/src/arch/x86/core/pcidirect.c @@ -45,6 +45,7 @@ void pcidirect_prepare ( struct pci_device *pci, int where ) { PCIDIRECT_CONFIG_ADDRESS ); } +PROVIDE_PCIAPI_INLINE ( direct, pci_can_probe ); PROVIDE_PCIAPI_INLINE ( direct, pci_discover ); PROVIDE_PCIAPI_INLINE ( direct, pci_read_config_byte ); PROVIDE_PCIAPI_INLINE ( direct, pci_read_config_word ); @@ -53,5 +54,4 @@ PROVIDE_PCIAPI_INLINE ( direct, pci_write_config_byte ); PROVIDE_PCIAPI_INLINE ( direct, pci_write_config_word ); PROVIDE_PCIAPI_INLINE ( direct, pci_write_config_dword ); PROVIDE_PCIAPI_INLINE ( direct, pci_ioremap ); - -struct pci_api pcidirect_api = PCIAPI_RUNTIME ( direct ); +PROVIDE_PCIAPI_RUNTIME ( direct, PCIAPI_PRIORITY_DIRECT ); diff --git a/src/arch/x86/core/rdrand.c b/src/arch/x86/core/rdrand.c index 850ab1f11..05fc3cd23 100644 --- a/src/arch/x86/core/rdrand.c +++ b/src/arch/x86/core/rdrand.c @@ -22,6 +22,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /** @file * diff --git a/src/arch/x86/core/relocate.c b/src/arch/x86/core/relocate.c index 765d46560..3cdc53c2e 100644 --- a/src/arch/x86/core/relocate.c +++ b/src/arch/x86/core/relocate.c @@ -1,4 +1,5 @@ -#include <ipxe/io.h> +#include <ipxe/uaccess.h> +#include <ipxe/memmap.h> #include <registers.h> /* @@ -41,82 +42,73 @@ extern char _etextdata[]; * to the prefix in %edi. */ __asmcall void relocate ( struct i386_all_regs *ix86 ) { - struct memory_map memmap; - uint32_t start, end, size, padded_size, max; - uint32_t new_start, new_end; - unsigned i; + struct memmap_region region; + physaddr_t start, end, max; + physaddr_t new_start, new_end; + physaddr_t r_start, r_end; + size_t size, padded_size; - /* Get memory map and current location */ - get_memmap ( &memmap ); + /* Show whole memory map (for debugging) */ + memmap_dump_all ( 0 ); + + /* Get current location */ start = virt_to_phys ( _textdata ); end = virt_to_phys ( _etextdata ); size = ( end - start ); padded_size = ( size + ALIGN - 1 ); - DBG ( "Relocate: currently at [%x,%x)\n" - "...need %x bytes for %d-byte alignment\n", - start, end, padded_size, ALIGN ); + DBGC ( ®ion, "Relocate: currently at [%#08lx,%#08lx)\n" + "...need %#zx bytes for %d-byte alignment\n", + start, end, padded_size, ALIGN ); /* Determine maximum usable address */ max = MAX_ADDR; if ( ix86->regs.ebp < max ) { max = ix86->regs.ebp; - DBG ( "Limiting relocation to [0,%x)\n", max ); + DBGC ( ®ion, "Limiting relocation to [0,%#08lx)\n", max ); } /* Walk through the memory map and find the highest address - * below 4GB that iPXE will fit into. + * above the current iPXE and below 4GB that iPXE will fit + * into. */ new_end = end; - for ( i = 0 ; i < memmap.count ; i++ ) { - struct memory_region *region = &memmap.regions[i]; - uint32_t r_start, r_end; + for_each_memmap_from ( ®ion, end, 0 ) { - DBG ( "Considering [%llx,%llx)\n", region->start, region->end); - /* Truncate block to maximum address. This will be - * less than 4GB, which means that we can get away - * with using just 32-bit arithmetic after this stage. + * strictly less than 4GB, which means that we can get + * away with using just 32-bit arithmetic after this + * stage. */ - if ( region->start > max ) { - DBG ( "...starts after max=%x\n", max ); + DBGC_MEMMAP ( ®ion, ®ion ); + if ( region.min > max ) { + DBGC ( ®ion, "...starts after max=%#08lx\n", max ); + break; + } + r_start = region.min; + if ( ! memmap_is_usable ( ®ion ) ) { + DBGC ( ®ion, "...not usable\n" ); continue; } - r_start = region->start; - if ( region->end > max ) { - DBG ( "...end truncated to max=%x\n", max ); + r_end = ( r_start + memmap_size ( ®ion ) ); + if ( ( r_end == 0 ) || ( r_end > max ) ) { + DBGC ( ®ion, "...end truncated to max=%#08lx\n", + max ); r_end = max; - } else { - r_end = region->end; - } - DBG ( "...usable portion is [%x,%x)\n", r_start, r_end ); - - /* If we have rounded down r_end below r_ start, skip - * this block. - */ - if ( r_end < r_start ) { - DBG ( "...truncated to negative size\n" ); - continue; } + DBGC ( ®ion, "...usable portion is [%#08lx,%#08lx)\n", + r_start, r_end ); /* Check that there is enough space to fit in iPXE */ - if ( ( r_end - r_start ) < size ) { - DBG ( "...too small (need %x bytes)\n", size ); + if ( ( r_end - r_start ) < padded_size ) { + DBGC ( ®ion, "...too small (need %#zx bytes)\n", + padded_size ); continue; } - /* If the start address of the iPXE we would - * place in this block is higher than the end address - * of the current highest block, use this block. - * - * Note that this avoids overlaps with the current - * iPXE, as well as choosing the highest of all viable - * blocks. - */ - if ( ( r_end - size ) > new_end ) { - new_end = r_end; - DBG ( "...new best block found.\n" ); - } + /* Use highest block with enough space */ + new_end = r_end; + DBGC ( ®ion, "...new best block found.\n" ); } /* Calculate new location of iPXE, and align it to the @@ -126,9 +118,9 @@ __asmcall void relocate ( struct i386_all_regs *ix86 ) { new_start += ( ( start - new_start ) & ( ALIGN - 1 ) ); new_end = new_start + size; - DBG ( "Relocating from [%x,%x) to [%x,%x)\n", - start, end, new_start, new_end ); - + DBGC ( ®ion, "Relocating from [%#08lx,%#08lx) to [%#08lx,%#08lx)\n", + start, end, new_start, new_end ); + /* Let prefix know what to copy */ ix86->regs.esi = start; ix86->regs.edi = new_start; diff --git a/src/arch/x86/core/runtime.c b/src/arch/x86/core/runtime.c index 02072b5bf..86083b1f9 100644 --- a/src/arch/x86/core/runtime.c +++ b/src/arch/x86/core/runtime.c @@ -32,6 +32,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stddef.h> #include <stdint.h> #include <stdlib.h> +#include <string.h> #include <ctype.h> #include <errno.h> #include <assert.h> @@ -69,6 +70,7 @@ static void cmdline_image_free ( struct refcnt *refcnt ) { struct image *image = container_of ( refcnt, struct image, refcnt ); DBGC ( image, "RUNTIME freeing command line\n" ); + free_image ( refcnt ); free ( cmdline_copy ); } @@ -76,6 +78,7 @@ static void cmdline_image_free ( struct refcnt *refcnt ) { static struct image cmdline_image = { .refcnt = REF_INIT ( cmdline_image_free ), .name = "<CMDLINE>", + .flags = ( IMAGE_STATIC | IMAGE_STATIC_NAME ), .type = &script_image_type, }; @@ -114,9 +117,7 @@ static void cmdline_strip ( char *cmdline, const char *cruft ) { * @ret rc Return status code */ static int cmdline_init ( void ) { - userptr_t cmdline_user; char *cmdline; - size_t len; int rc; /* Do nothing if no command line was specified */ @@ -124,19 +125,15 @@ static int cmdline_init ( void ) { DBGC ( colour, "RUNTIME found no command line\n" ); return 0; } - cmdline_user = phys_to_user ( cmdline_phys ); - len = ( strlen_user ( cmdline_user, 0 ) + 1 /* NUL */ ); /* Allocate and copy command line */ - cmdline_copy = malloc ( len ); + cmdline_copy = strdup ( phys_to_virt ( cmdline_phys ) ); if ( ! cmdline_copy ) { - DBGC ( colour, "RUNTIME could not allocate %zd bytes for " - "command line\n", len ); + DBGC ( colour, "RUNTIME could not allocate command line\n" ); rc = -ENOMEM; goto err_alloc_cmdline_copy; } cmdline = cmdline_copy; - copy_from_user ( cmdline, cmdline_user, 0, len ); DBGC ( colour, "RUNTIME found command line \"%s\" at %08x\n", cmdline, cmdline_phys ); @@ -151,7 +148,7 @@ static int cmdline_init ( void ) { DBGC ( colour, "RUNTIME using command line \"%s\"\n", cmdline ); /* Prepare and register image */ - cmdline_image.data = virt_to_user ( cmdline ); + cmdline_image.data = cmdline; cmdline_image.len = strlen ( cmdline ); if ( cmdline_image.len ) { if ( ( rc = register_image ( &cmdline_image ) ) != 0 ) { @@ -193,7 +190,7 @@ static int initrd_init ( void ) { initrd_phys, ( initrd_phys + initrd_len ) ); /* Create initrd image */ - image = image_memory ( "<INITRD>", phys_to_user ( initrd_phys ), + image = image_memory ( "<INITRD>", phys_to_virt ( initrd_phys ), initrd_len ); if ( ! image ) { DBGC ( colour, "RUNTIME could not create initrd image\n" ); diff --git a/src/arch/x86/core/video_subr.c b/src/arch/x86/core/video_subr.c index f5cc4cdd4..4e9ef466f 100644 --- a/src/arch/x86/core/video_subr.c +++ b/src/arch/x86/core/video_subr.c @@ -109,5 +109,6 @@ struct console_driver vga_console __console_driver = { }; struct init_fn video_init_fn __init_fn ( INIT_EARLY ) = { + .name = "video", .initialise = video_init, }; diff --git a/src/arch/x86/core/vram_settings.c b/src/arch/x86/core/vram_settings.c index 9c169b40c..a97a463fe 100644 --- a/src/arch/x86/core/vram_settings.c +++ b/src/arch/x86/core/vram_settings.c @@ -23,6 +23,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +#include <string.h> #include <ipxe/uaccess.h> #include <ipxe/settings.h> @@ -47,12 +48,12 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * @ret len Length of setting data, or negative error */ static int vram_fetch ( void *data, size_t len ) { - userptr_t vram = phys_to_user ( VRAM_BASE ); + const void *vram = phys_to_virt ( VRAM_BASE ); /* Copy video RAM */ if ( len > VRAM_LEN ) len = VRAM_LEN; - copy_from_user ( data, vram, 0, len ); + memcpy ( data, vram, len ); return VRAM_LEN; } diff --git a/src/arch/x86/core/x86_bigint.c b/src/arch/x86/core/x86_bigint.c deleted file mode 100644 index 74e5da9a2..000000000 --- a/src/arch/x86/core/x86_bigint.c +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - * - * You can also choose to distribute this program under the terms of - * the Unmodified Binary Distribution Licence (as given in the file - * COPYING.UBDL), provided that you have satisfied its requirements. - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <stdint.h> -#include <string.h> -#include <ipxe/bigint.h> - -/** @file - * - * Big integer support - */ - -/** - * Multiply big integers - * - * @v multiplicand0 Element 0 of big integer to be multiplied - * @v multiplicand_size Number of elements in multiplicand - * @v multiplier0 Element 0 of big integer to be multiplied - * @v multiplier_size Number of elements in multiplier - * @v result0 Element 0 of big integer to hold result - */ -void bigint_multiply_raw ( const uint32_t *multiplicand0, - unsigned int multiplicand_size, - const uint32_t *multiplier0, - unsigned int multiplier_size, - uint32_t *result0 ) { - unsigned int result_size = ( multiplicand_size + multiplier_size ); - const bigint_t ( multiplicand_size ) __attribute__ (( may_alias )) - *multiplicand = ( ( const void * ) multiplicand0 ); - const bigint_t ( multiplier_size ) __attribute__ (( may_alias )) - *multiplier = ( ( const void * ) multiplier0 ); - bigint_t ( result_size ) __attribute__ (( may_alias )) - *result = ( ( void * ) result0 ); - unsigned int i; - unsigned int j; - uint32_t multiplicand_element; - uint32_t multiplier_element; - uint32_t *result_elements; - uint32_t discard_a; - uint32_t discard_d; - long index; - - /* Zero result */ - memset ( result, 0, sizeof ( *result ) ); - - /* Multiply integers one element at a time */ - for ( i = 0 ; i < multiplicand_size ; i++ ) { - multiplicand_element = multiplicand->element[i]; - for ( j = 0 ; j < multiplier_size ; j++ ) { - multiplier_element = multiplier->element[j]; - result_elements = &result->element[ i + j ]; - /* Perform a single multiply, and add the - * resulting double-element into the result, - * carrying as necessary. The carry can - * never overflow beyond the end of the - * result, since: - * - * a < 2^{n}, b < 2^{m} => ab < 2^{n+m} - */ - __asm__ __volatile__ ( "mull %5\n\t" - "addl %%eax, (%6,%2,4)\n\t" - "adcl %%edx, 4(%6,%2,4)\n\t" - "\n1:\n\t" - "adcl $0, 8(%6,%2,4)\n\t" - "inc %2\n\t" - /* Does not affect CF */ - "jc 1b\n\t" - : "=&a" ( discard_a ), - "=&d" ( discard_d ), - "=&r" ( index ), - "+m" ( *result ) - : "0" ( multiplicand_element ), - "g" ( multiplier_element ), - "r" ( result_elements ), - "2" ( 0 ) ); - } - } -} diff --git a/src/arch/x86/core/x86_io.c b/src/arch/x86/core/x86_io.c index 6c6b6e1e7..270ed7bef 100644 --- a/src/arch/x86/core/x86_io.c +++ b/src/arch/x86/core/x86_io.c @@ -32,6 +32,69 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * */ +/** Threshold for port I/O-mapped addresses + * + * On x86, port I/O instructions (inb/outb/etc) can take only an 8-bit + * or 16-bit address (in %dx). All I/O ports must therefore have a + * value in the first 64kB of the address space. + * + * Virtual addresses below 64kB can never be MMIO addresses: + * + * - In the UEFI memory model and x86_64 BIOS memory model, virtual + * addresses below 64kB are identity-mapped to the corresponding + * physical address. Since the first 64kB of address space is + * always RAM, no MMIO device can exist within this region. + * + * - In the i386 BIOS memory model, virtual addresses below 64kB cover + * the iPXE binary itself (which starts at address zero). Since the + * size of .textdata can never realistically be below 64kB (not + * least since the heap alone is 512kB), and since iPXE is placed + * into RAM as a contiguous block, no MMIO device can exist within + * this region. + * + * We therefore know that any (virtual) address returned by ioremap() + * must be outside the first 64kB of the address space. We can + * therefore use this as a threshold to determine whether a given + * address is a port I/O address or an MMIO address. + */ +#define PIO_THRESHOLD 0x10000 + +/** + * Read from I/O-mapped or memory-mapped device + * + * @v io_addr I/O address + * @ret data Value read + */ +#define X86_IOREADX( _api_func, _suffix, _type ) \ +static _type x86_ ## _api_func ( volatile _type *io_addr ) { \ + if ( ( ( intptr_t ) io_addr ) < PIO_THRESHOLD ) { \ + return in ## _suffix ( io_addr ); \ + } else { \ + return read ## _suffix ( io_addr ); \ + } \ +} +X86_IOREADX ( ioread8, b, uint8_t ); +X86_IOREADX ( ioread16, w, uint16_t ); +X86_IOREADX ( ioread32, l, uint32_t ); + +/** + * Write to I/O-mapped or memory-mapped device + * + * @v data Value to write + * @v io_addr I/O address + */ +#define X86_IOWRITEX( _api_func, _suffix, _type ) \ +static void x86_ ## _api_func ( _type data, volatile _type *io_addr ) { \ + if ( ( ( intptr_t ) io_addr ) < PIO_THRESHOLD ) { \ + out ## _suffix ( data, io_addr ); \ + } else { \ + write ## _suffix ( data, io_addr ); \ + } \ +} +X86_IOWRITEX ( iowrite8, b, uint8_t ); +X86_IOWRITEX ( iowrite16, w, uint16_t ); +X86_IOWRITEX ( iowrite32, l, uint32_t ); + /** * Read 64-bit qword from memory-mapped device * @@ -101,3 +164,9 @@ PROVIDE_IOAPI_INLINE ( x86, writeq ); PROVIDE_IOAPI ( x86, readq, i386_readq ); PROVIDE_IOAPI ( x86, writeq, i386_writeq ); #endif +PROVIDE_IOAPI ( x86, ioread8, x86_ioread8 ); +PROVIDE_IOAPI ( x86, ioread16, x86_ioread16 ); +PROVIDE_IOAPI ( x86, ioread32, x86_ioread32 ); +PROVIDE_IOAPI ( x86, iowrite8, x86_iowrite8 ); +PROVIDE_IOAPI ( x86, iowrite16, x86_iowrite16 ); +PROVIDE_IOAPI ( x86, iowrite32, x86_iowrite32 ); diff --git a/src/arch/x86/core/x86_string.c b/src/arch/x86/core/x86_string.c index 1a1e79dac..923552f66 100644 --- a/src/arch/x86/core/x86_string.c +++ b/src/arch/x86/core/x86_string.c @@ -28,6 +28,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <string.h> #include <config/defaults.h> diff --git a/src/arch/x86/core/x86_tcpip.c b/src/arch/x86/core/x86_tcpip.c index ed323d5d0..b3bfe2546 100644 --- a/src/arch/x86/core/x86_tcpip.c +++ b/src/arch/x86/core/x86_tcpip.c @@ -22,6 +22,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /** @file * diff --git a/src/arch/x86/core/x86_uart.c b/src/arch/x86/core/x86_uart.c index e455775bf..03809ff9b 100644 --- a/src/arch/x86/core/x86_uart.c +++ b/src/arch/x86/core/x86_uart.c @@ -1,5 +1,5 @@ /* - * Copyright (C) 2014 Michael Brown <mbrown@fensystems.co.uk>. + * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>. * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -29,41 +29,47 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * */ -#include <errno.h> -#include <ipxe/uart.h> +#include <string.h> +#include <ipxe/serial.h> +#include <ipxe/ns16550.h> -/** UART port bases */ -static uint16_t uart_base[] = { - [COM1] = 0x3f8, - [COM2] = 0x2f8, - [COM3] = 0x3e8, - [COM4] = 0x2e8, -}; +/** Define a fixed ISA UART */ +#define ISA_UART( NAME, BASE ) \ + static struct ns16550_uart ns16550_ ## NAME = { \ + .base = ( ( void * ) (BASE) ), \ + .clock = NS16550_CLK_DEFAULT, \ + }; \ + struct uart NAME = { \ + .refcnt = REF_INIT ( ref_no_free ), \ + .name = #NAME, \ + .op = &ns16550_operations, \ + .priv = &ns16550_ ## NAME, \ + } + +/* Fixed ISA UARTs */ +ISA_UART ( com1, COM1_BASE ); +ISA_UART ( com2, COM2_BASE ); +ISA_UART ( com3, COM3_BASE ); +ISA_UART ( com4, COM4_BASE ); /** - * Select UART port + * Register fixed ISA UARTs * - * @v uart UART - * @v port Port number, or 0 to disable * @ret rc Return status code */ -int uart_select ( struct uart *uart, unsigned int port ) { +int uart_register_fixed ( void ) { + static struct uart *ports[] = { COM1, COM2, COM3, COM4 }; + unsigned int i; int rc; - /* Set new UART base */ - if ( port >= ( sizeof ( uart_base ) / sizeof ( uart_base[0] ) ) ) { - rc = -ENODEV; - goto err; + /* Register all fixed ISA UARTs */ + for ( i = 0 ; i < ( sizeof ( ports ) / sizeof ( ports[0] ) ) ; i++ ) { + if ( ( rc = uart_register ( ports[i] ) ) != 0 ) { + DBGC ( ports[i], "UART could not register %s: %s\n", + ports[i]->name, strerror ( rc ) ); + return rc; + } } - uart->base = ( ( void * ) ( intptr_t ) uart_base[port] ); - - /* Check that UART exists */ - if ( ( rc = uart_exists ( uart ) ) != 0 ) - goto err; return 0; - - err: - uart->base = NULL; - return rc; } diff --git a/src/arch/x86/drivers/net/undiisr.S b/src/arch/x86/drivers/net/undiisr.S index 8ba5c5354..0b07cb396 100644 --- a/src/arch/x86/drivers/net/undiisr.S +++ b/src/arch/x86/drivers/net/undiisr.S @@ -33,8 +33,16 @@ undiisr: /* Check that we have an UNDI entry point */ cmpw $0, undinet_entry_point je chain - + + /* Mask interrupt and set rearm flag */ + movw undiisr_imr, %dx + inb %dx, %al + orb undiisr_bit, %al + outb %al, %dx + movb %al, undiisr_rearm + /* Issue UNDI API call */ + movw %ds, %ax movw %ax, %es movw $undinet_params, %di movw $PXENV_UNDI_ISR, %bx diff --git a/src/arch/x86/drivers/net/undinet.c b/src/arch/x86/drivers/net/undinet.c index 43cb18bfe..f4f78432a 100644 --- a/src/arch/x86/drivers/net/undinet.c +++ b/src/arch/x86/drivers/net/undinet.c @@ -373,6 +373,18 @@ extern void undiisr ( void ); uint8_t __data16 ( undiisr_irq ); #define undiisr_irq __use_data16 ( undiisr_irq ) +/** IRQ mask register */ +uint16_t __data16 ( undiisr_imr ); +#define undiisr_imr __use_data16 ( undiisr_imr ) + +/** IRQ mask bit */ +uint8_t __data16 ( undiisr_bit ); +#define undiisr_bit __use_data16 ( undiisr_bit ) + +/** IRQ rearm flag */ +uint8_t __data16 ( undiisr_rearm ); +#define undiisr_rearm __use_data16 ( undiisr_rearm ) + /** IRQ chain vector */ struct segoff __data16 ( undiisr_next_handler ); #define undiisr_next_handler __use_data16 ( undiisr_next_handler ) @@ -395,6 +407,9 @@ static void undinet_hook_isr ( unsigned int irq ) { assert ( undiisr_irq == 0 ); undiisr_irq = irq; + undiisr_imr = IMR_REG ( irq ); + undiisr_bit = IMR_BIT ( irq ); + undiisr_rearm = 0; hook_bios_interrupt ( IRQ_INT ( irq ), ( ( intptr_t ) undiisr ), &undiisr_next_handler ); } @@ -588,6 +603,14 @@ static void undinet_poll ( struct net_device *netdev ) { * support interrupts. */ if ( ! undinet_isr_triggered() ) { + + /* Rearm interrupt if needed */ + if ( undiisr_rearm ) { + undiisr_rearm = 0; + assert ( undinic->irq != 0 ); + enable_irq ( undinic->irq ); + } + /* Allow interrupt to occur */ profile_start ( &undinet_irq_profiler ); __asm__ __volatile__ ( "sti\n\t" @@ -838,15 +861,19 @@ static const struct undinet_irq_broken undinet_irq_broken_list[] = { { 0x8086, 0x1503, PCI_ANY_ID, PCI_ANY_ID }, /* HP 745 G3 laptop */ { 0x14e4, 0x1687, PCI_ANY_ID, PCI_ANY_ID }, + /* ASUSTeK KNPA-U16 server */ + { 0x8086, 0x1521, 0x1043, PCI_ANY_ID }, }; /** * Check for devices with broken support for generating interrupts * - * @v desc Device description + * @v netdev Net device * @ret irq_is_broken Interrupt support is broken; no interrupts are generated */ -static int undinet_irq_is_broken ( struct device_description *desc ) { +static int undinet_irq_is_broken ( struct net_device *netdev ) { + struct undi_nic *undinic = netdev->priv; + struct device_description *desc = &netdev->dev->desc; const struct undinet_irq_broken *broken; struct pci_device pci; uint16_t subsys_vendor; @@ -872,9 +899,25 @@ static int undinet_irq_is_broken ( struct device_description *desc ) { ( broken->pci_subsys_vendor == PCI_ANY_ID ) ) && ( ( broken->pci_subsys == subsys ) || ( broken->pci_subsys == PCI_ANY_ID ) ) ) { + DBGC ( undinic, "UNDINIC %p %04x:%04x subsys " + "%04x:%04x has broken interrupts\n", + undinic, desc->vendor, desc->device, + subsys_vendor, subsys ); return 1; } } + + /* Check for a PCI Express capability. Given the number of + * issues found with legacy INTx emulation on PCIe systems, we + * assume that there is a high chance of interrupts not + * working on any PCIe device. + */ + if ( pci_find_capability ( &pci, PCI_CAP_ID_EXP ) ) { + DBGC ( undinic, "UNDINIC %p is PCI Express: assuming " + "interrupts are unreliable\n", undinic ); + return 1; + } + return 0; } @@ -972,6 +1015,10 @@ int undinet_probe ( struct undi_device *undi, struct device *dev ) { } DBGC ( undinic, "UNDINIC %p has MAC address %s and IRQ %d\n", undinic, eth_ntoa ( netdev->hw_addr ), undinic->irq ); + if ( undinic->irq ) { + /* Sanity check - prefix should have disabled the IRQ */ + assert ( ! irq_enabled ( undinic->irq ) ); + } /* Get interface information */ memset ( &undi_iface, 0, sizeof ( undi_iface ) ); @@ -993,7 +1040,7 @@ int undinet_probe ( struct undi_device *undi, struct device *dev ) { undinic ); undinic->hacks |= UNDI_HACK_EB54; } - if ( undinet_irq_is_broken ( &dev->desc ) ) { + if ( undinet_irq_is_broken ( netdev ) ) { DBGC ( undinic, "UNDINIC %p forcing polling mode due to " "broken interrupts\n", undinic ); undinic->irq_supported = 0; diff --git a/src/arch/x86/drivers/xen/hvm.c b/src/arch/x86/drivers/xen/hvm.c index b77cdd14c..cf41cc955 100644 --- a/src/arch/x86/drivers/xen/hvm.c +++ b/src/arch/x86/drivers/xen/hvm.c @@ -25,6 +25,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdint.h> #include <stdio.h> +#include <string.h> #include <errno.h> #include <ipxe/malloc.h> #include <ipxe/pci.h> diff --git a/src/arch/x86/hci/commands/cpuid_cmd.c b/src/arch/x86/hci/commands/cpuid_cmd.c index d73ce2a3e..f4d7305e8 100644 --- a/src/arch/x86/hci/commands/cpuid_cmd.c +++ b/src/arch/x86/hci/commands/cpuid_cmd.c @@ -22,6 +22,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <stdint.h> #include <stdio.h> @@ -95,7 +96,4 @@ static int cpuid_exec ( int argc, char **argv ) { } /** x86 CPU feature detection command */ -struct command cpuid_command __command = { - .name = "cpuid", - .exec = cpuid_exec, -}; +COMMAND ( cpuid, cpuid_exec ); diff --git a/src/arch/x86/hci/commands/pxe_cmd.c b/src/arch/x86/hci/commands/pxe_cmd.c index 473b97f97..cf1a36ed6 100644 --- a/src/arch/x86/hci/commands/pxe_cmd.c +++ b/src/arch/x86/hci/commands/pxe_cmd.c @@ -105,13 +105,5 @@ static int stoppxe_exec ( int argc __unused, char **argv __unused ) { } /** PXE commands */ -struct command pxe_commands[] __command = { - { - .name = "startpxe", - .exec = startpxe_exec, - }, - { - .name = "stoppxe", - .exec = stoppxe_exec, - }, -}; +COMMAND ( startpxe, startpxe_exec ); +COMMAND ( stoppxe, stoppxe_exec ); diff --git a/src/arch/x86/image/bzimage.c b/src/arch/x86/image/bzimage.c index 2c776147d..16a47fc57 100644 --- a/src/arch/x86/image/bzimage.c +++ b/src/arch/x86/image/bzimage.c @@ -32,12 +32,13 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdint.h> #include <stdlib.h> +#include <stdio.h> #include <string.h> #include <errno.h> #include <assert.h> #include <realmode.h> #include <bzimage.h> -#include <initrd.h> +#include <ipxe/initrd.h> #include <ipxe/uaccess.h> #include <ipxe/image.h> #include <ipxe/segment.h> @@ -56,7 +57,7 @@ struct bzimage_context { /** Real-mode kernel portion load segment address */ unsigned int rm_kernel_seg; /** Real-mode kernel portion load address */ - userptr_t rm_kernel; + void *rm_kernel; /** Real-mode kernel portion file size */ size_t rm_filesz; /** Real-mode heap top (offset from rm_kernel) */ @@ -68,7 +69,7 @@ struct bzimage_context { /** Real-mode kernel portion total memory size */ size_t rm_memsz; /** Non-real-mode kernel portion load address */ - userptr_t pm_kernel; + void *pm_kernel; /** Non-real-mode kernel portion file and memory size */ size_t pm_sz; /** Video mode */ @@ -76,14 +77,9 @@ struct bzimage_context { /** Memory limit */ uint64_t mem_limit; /** Initrd address */ - physaddr_t ramdisk_image; + void *initrd; /** Initrd size */ - physaddr_t ramdisk_size; - - /** Command line magic block */ - struct bzimage_cmdline cmdline_magic; - /** bzImage header */ - struct bzimage_header bzhdr; + physaddr_t initrd_size; }; /** @@ -91,35 +87,31 @@ struct bzimage_context { * * @v image bzImage file * @v bzimg bzImage context - * @v src bzImage to parse * @ret rc Return status code */ static int bzimage_parse_header ( struct image *image, - struct bzimage_context *bzimg, - userptr_t src ) { + struct bzimage_context *bzimg ) { + const struct bzimage_header *bzhdr; unsigned int syssize; int is_bzimage; + /* Initialise context */ + memset ( bzimg, 0, sizeof ( *bzimg ) ); + /* Sanity check */ - if ( image->len < ( BZI_HDR_OFFSET + sizeof ( bzimg->bzhdr ) ) ) { - DBGC ( image, "bzImage %p too short for kernel header\n", - image ); + if ( image->len < ( BZI_HDR_OFFSET + sizeof ( *bzhdr ) ) ) { + DBGC ( image, "bzImage %s too short for kernel header\n", + image->name ); return -ENOEXEC; } - - /* Read in header structures */ - memset ( bzimg, 0, sizeof ( *bzimg ) ); - copy_from_user ( &bzimg->cmdline_magic, src, BZI_CMDLINE_OFFSET, - sizeof ( bzimg->cmdline_magic ) ); - copy_from_user ( &bzimg->bzhdr, src, BZI_HDR_OFFSET, - sizeof ( bzimg->bzhdr ) ); + bzhdr = ( image->data + BZI_HDR_OFFSET ); /* Calculate size of real-mode portion */ - bzimg->rm_filesz = ( ( ( bzimg->bzhdr.setup_sects ? - bzimg->bzhdr.setup_sects : 4 ) + 1 ) << 9 ); + bzimg->rm_filesz = ( ( ( bzhdr->setup_sects ? + bzhdr->setup_sects : 4 ) + 1 ) << 9 ); if ( bzimg->rm_filesz > image->len ) { - DBGC ( image, "bzImage %p too short for %zd byte of setup\n", - image, bzimg->rm_filesz ); + DBGC ( image, "bzImage %s too short for %zd byte of setup\n", + image->name, bzimg->rm_filesz ); return -ENOEXEC; } bzimg->rm_memsz = BZI_ASSUMED_RM_SIZE; @@ -129,13 +121,14 @@ static int bzimage_parse_header ( struct image *image, syssize = ( ( bzimg->pm_sz + 15 ) / 16 ); /* Check for signatures and determine version */ - if ( bzimg->bzhdr.boot_flag != BZI_BOOT_FLAG ) { - DBGC ( image, "bzImage %p missing 55AA signature\n", image ); + if ( bzhdr->boot_flag != BZI_BOOT_FLAG ) { + DBGC ( image, "bzImage %s missing 55AA signature\n", + image->name ); return -ENOEXEC; } - if ( bzimg->bzhdr.header == BZI_SIGNATURE ) { + if ( bzhdr->header == BZI_SIGNATURE ) { /* 2.00+ */ - bzimg->version = bzimg->bzhdr.version; + bzimg->version = bzhdr->version; } else { /* Pre-2.00. Check that the syssize field is correct, * as a guard against accepting arbitrary binary data, @@ -145,20 +138,21 @@ static int bzimage_parse_header ( struct image *image, * check this field. */ bzimg->version = 0x0100; - if ( bzimg->bzhdr.syssize != syssize ) { - DBGC ( image, "bzImage %p bad syssize %x (expected " - "%x)\n", image, bzimg->bzhdr.syssize, syssize ); + if ( bzhdr->syssize != syssize ) { + DBGC ( image, "bzImage %s bad syssize %x (expected " + "%x)\n", image->name, bzhdr->syssize, + syssize ); return -ENOEXEC; } } /* Determine image type */ is_bzimage = ( ( bzimg->version >= 0x0200 ) ? - ( bzimg->bzhdr.loadflags & BZI_LOAD_HIGH ) : 0 ); + ( bzhdr->loadflags & BZI_LOAD_HIGH ) : 0 ); /* Calculate load address of real-mode portion */ bzimg->rm_kernel_seg = ( is_bzimage ? 0x1000 : 0x9000 ); - bzimg->rm_kernel = real_to_user ( bzimg->rm_kernel_seg, 0 ); + bzimg->rm_kernel = real_to_virt ( bzimg->rm_kernel_seg, 0 ); /* Allow space for the stack and heap */ bzimg->rm_memsz += BZI_STACK_SIZE; @@ -169,24 +163,24 @@ static int bzimage_parse_header ( struct image *image, bzimg->rm_memsz += BZI_CMDLINE_SIZE; /* Calculate load address of protected-mode portion */ - bzimg->pm_kernel = phys_to_user ( is_bzimage ? BZI_LOAD_HIGH_ADDR + bzimg->pm_kernel = phys_to_virt ( is_bzimage ? BZI_LOAD_HIGH_ADDR : BZI_LOAD_LOW_ADDR ); /* Extract video mode */ - bzimg->vid_mode = bzimg->bzhdr.vid_mode; + bzimg->vid_mode = bzhdr->vid_mode; /* Extract memory limit */ bzimg->mem_limit = ( ( bzimg->version >= 0x0203 ) ? - bzimg->bzhdr.initrd_addr_max : BZI_INITRD_MAX ); + bzhdr->initrd_addr_max : BZI_INITRD_MAX ); /* Extract command line size */ bzimg->cmdline_size = ( ( bzimg->version >= 0x0206 ) ? - bzimg->bzhdr.cmdline_size : BZI_CMDLINE_SIZE ); + bzhdr->cmdline_size : BZI_CMDLINE_SIZE ); - DBGC ( image, "bzImage %p version %04x RM %#lx+%#zx PM %#lx+%#zx " - "cmdlen %zd\n", image, bzimg->version, - user_to_phys ( bzimg->rm_kernel, 0 ), bzimg->rm_filesz, - user_to_phys ( bzimg->pm_kernel, 0 ), bzimg->pm_sz, + DBGC ( image, "bzImage %s version %04x RM %#lx+%#zx PM %#lx+%#zx " + "cmdlen %zd\n", image->name, bzimg->version, + virt_to_phys ( bzimg->rm_kernel ), bzimg->rm_filesz, + virt_to_phys ( bzimg->pm_kernel ), bzimg->pm_sz, bzimg->cmdline_size ); return 0; @@ -197,49 +191,44 @@ static int bzimage_parse_header ( struct image *image, * * @v image bzImage file * @v bzimg bzImage context - * @v dst bzImage to update */ static void bzimage_update_header ( struct image *image, - struct bzimage_context *bzimg, - userptr_t dst ) { + struct bzimage_context *bzimg ) { + struct bzimage_header *bzhdr = ( bzimg->rm_kernel + BZI_HDR_OFFSET ); + struct bzimage_cmdline *cmdline; /* Set loader type */ if ( bzimg->version >= 0x0200 ) - bzimg->bzhdr.type_of_loader = BZI_LOADER_TYPE_IPXE; + bzhdr->type_of_loader = BZI_LOADER_TYPE_IPXE; /* Set heap end pointer */ if ( bzimg->version >= 0x0201 ) { - bzimg->bzhdr.heap_end_ptr = ( bzimg->rm_heap - 0x200 ); - bzimg->bzhdr.loadflags |= BZI_CAN_USE_HEAP; + bzhdr->heap_end_ptr = ( bzimg->rm_heap - 0x200 ); + bzhdr->loadflags |= BZI_CAN_USE_HEAP; } /* Set command line */ if ( bzimg->version >= 0x0202 ) { - bzimg->bzhdr.cmd_line_ptr = user_to_phys ( bzimg->rm_kernel, - bzimg->rm_cmdline ); + bzhdr->cmd_line_ptr = ( virt_to_phys ( bzimg->rm_kernel ) + + bzimg->rm_cmdline ); } else { - bzimg->cmdline_magic.magic = BZI_CMDLINE_MAGIC; - bzimg->cmdline_magic.offset = bzimg->rm_cmdline; + cmdline = ( bzimg->rm_kernel + BZI_CMDLINE_OFFSET ); + cmdline->magic = BZI_CMDLINE_MAGIC; + cmdline->offset = bzimg->rm_cmdline; if ( bzimg->version >= 0x0200 ) - bzimg->bzhdr.setup_move_size = bzimg->rm_memsz; + bzhdr->setup_move_size = bzimg->rm_memsz; } /* Set video mode */ - bzimg->bzhdr.vid_mode = bzimg->vid_mode; + bzhdr->vid_mode = bzimg->vid_mode; + DBGC ( image, "bzImage %s vidmode %d\n", + image->name, bzhdr->vid_mode ); /* Set initrd address */ if ( bzimg->version >= 0x0200 ) { - bzimg->bzhdr.ramdisk_image = bzimg->ramdisk_image; - bzimg->bzhdr.ramdisk_size = bzimg->ramdisk_size; + bzhdr->ramdisk_image = virt_to_phys ( bzimg->initrd ); + bzhdr->ramdisk_size = bzimg->initrd_size; } - - /* Write out header structures */ - copy_to_user ( dst, BZI_CMDLINE_OFFSET, &bzimg->cmdline_magic, - sizeof ( bzimg->cmdline_magic ) ); - copy_to_user ( dst, BZI_HDR_OFFSET, &bzimg->bzhdr, - sizeof ( bzimg->bzhdr ) ); - - DBGC ( image, "bzImage %p vidmode %d\n", image, bzimg->vid_mode ); } /** @@ -270,8 +259,9 @@ static int bzimage_parse_cmdline ( struct image *image, } else { bzimg->vid_mode = strtoul ( vga, &end, 0 ); if ( *end ) { - DBGC ( image, "bzImage %p strange \"vga=\" " - "terminator '%c'\n", image, *end ); + DBGC ( image, "bzImage %s strange \"vga=\" " + "terminator '%c'\n", + image->name, *end ); } } if ( sep ) @@ -298,8 +288,8 @@ static int bzimage_parse_cmdline ( struct image *image, case ' ': break; default: - DBGC ( image, "bzImage %p strange \"mem=\" " - "terminator '%c'\n", image, *end ); + DBGC ( image, "bzImage %s strange \"mem=\" " + "terminator '%c'\n", image->name, *end ); break; } bzimg->mem_limit -= 1; @@ -317,76 +307,13 @@ static int bzimage_parse_cmdline ( struct image *image, static void bzimage_set_cmdline ( struct image *image, struct bzimage_context *bzimg ) { const char *cmdline = ( image->cmdline ? image->cmdline : "" ); - size_t cmdline_len; + char *rm_cmdline; /* Copy command line down to real-mode portion */ - cmdline_len = ( strlen ( cmdline ) + 1 ); - if ( cmdline_len > bzimg->cmdline_size ) - cmdline_len = bzimg->cmdline_size; - copy_to_user ( bzimg->rm_kernel, bzimg->rm_cmdline, - cmdline, cmdline_len ); - DBGC ( image, "bzImage %p command line \"%s\"\n", image, cmdline ); -} - -/** - * Align initrd length - * - * @v len Length - * @ret len Length rounded up to INITRD_ALIGN - */ -static inline size_t bzimage_align ( size_t len ) { - - return ( ( len + INITRD_ALIGN - 1 ) & ~( INITRD_ALIGN - 1 ) ); -} - -/** - * Load initrd - * - * @v image bzImage image - * @v initrd initrd image - * @v address Address at which to load, or UNULL - * @ret len Length of loaded image, excluding zero-padding - */ -static size_t bzimage_load_initrd ( struct image *image, - struct image *initrd, - userptr_t address ) { - const char *filename = cpio_name ( initrd ); - struct cpio_header cpio; - size_t offset; - size_t pad_len; - - /* Skip hidden images */ - if ( initrd->flags & IMAGE_HIDDEN ) - return 0; - - /* Create cpio header for non-prebuilt images */ - offset = cpio_header ( initrd, &cpio ); - - /* Copy in initrd image body (and cpio header if applicable) */ - if ( address ) { - memmove_user ( address, offset, initrd->data, 0, initrd->len ); - if ( offset ) { - memset_user ( address, 0, 0, offset ); - copy_to_user ( address, 0, &cpio, sizeof ( cpio ) ); - copy_to_user ( address, sizeof ( cpio ), filename, - cpio_name_len ( initrd ) ); - } - DBGC ( image, "bzImage %p initrd %p [%#08lx,%#08lx,%#08lx)" - "%s%s\n", image, initrd, user_to_phys ( address, 0 ), - user_to_phys ( address, offset ), - user_to_phys ( address, ( offset + initrd->len ) ), - ( filename ? " " : "" ), ( filename ? filename : "" ) ); - DBGC2_MD5A ( image, user_to_phys ( address, offset ), - user_to_virt ( address, offset ), initrd->len ); - } - offset += initrd->len; - - /* Zero-pad to next INITRD_ALIGN boundary */ - pad_len = ( ( -offset ) & ( INITRD_ALIGN - 1 ) ); - if ( address ) - memset_user ( address, offset, 0, pad_len ); - - return offset; + rm_cmdline = ( bzimg->rm_kernel + bzimg->rm_cmdline ); + snprintf ( rm_cmdline, bzimg->cmdline_size, "%s", cmdline ); + DBGC ( image, "bzImage %s command line \"%s\"\n", + image->name, rm_cmdline ); } /** @@ -398,48 +325,52 @@ static size_t bzimage_load_initrd ( struct image *image, */ static int bzimage_check_initrds ( struct image *image, struct bzimage_context *bzimg ) { - struct image *initrd; - userptr_t bottom; - size_t len = 0; + struct memmap_region region; + physaddr_t min; + physaddr_t max; + physaddr_t dest; int rc; /* Calculate total loaded length of initrds */ - for_each_image ( initrd ) { - - /* Calculate length */ - len += bzimage_load_initrd ( image, initrd, UNULL ); - len = bzimage_align ( len ); - - DBGC ( image, "bzImage %p initrd %p from [%#08lx,%#08lx)%s%s\n", - image, initrd, user_to_phys ( initrd->data, 0 ), - user_to_phys ( initrd->data, initrd->len ), - ( initrd->cmdline ? " " : "" ), - ( initrd->cmdline ? initrd->cmdline : "" ) ); - DBGC2_MD5A ( image, user_to_phys ( initrd->data, 0 ), - user_to_virt ( initrd->data, 0 ), initrd->len ); - } + bzimg->initrd_size = initrd_len(); - /* Calculate lowest usable address */ - bottom = userptr_add ( bzimg->pm_kernel, bzimg->pm_sz ); + /* Succeed if there are no initrds */ + if ( ! bzimg->initrd_size ) + return 0; - /* Check that total length fits within space available for - * reshuffling. This is a conservative check, since CPIO - * headers are not present during reshuffling, but this - * doesn't hurt and keeps the code simple. - */ - if ( ( rc = initrd_reshuffle_check ( len, bottom ) ) != 0 ) { - DBGC ( image, "bzImage %p failed reshuffle check: %s\n", - image, strerror ( rc ) ); + /* Calculate available load region after reshuffling */ + if ( ( rc = initrd_region ( bzimg->initrd_size, ®ion ) ) != 0 ) { + DBGC ( image, "bzImage %s no region for initrds: %s\n", + image->name, strerror ( rc ) ); return rc; } - /* Check that total length fits within kernel's memory limit */ - if ( user_to_phys ( bottom, len ) > bzimg->mem_limit ) { - DBGC ( image, "bzImage %p not enough space for initrds\n", - image ); + /* Limit region to avoiding kernel itself */ + min = virt_to_phys ( bzimg->pm_kernel + bzimg->pm_sz ); + if ( min < region.min ) + min = region.min; + + /* Limit region to kernel's memory limit */ + max = region.max; + if ( max > bzimg->mem_limit ) + max = bzimg->mem_limit; + + /* Calculate installation address */ + if ( max < ( bzimg->initrd_size - 1 ) ) { + DBGC ( image, "bzImage %s not enough space for initrds\n", + image->name ); + return -ENOBUFS; + } + dest = ( ( max + 1 - bzimg->initrd_size ) & ~( INITRD_ALIGN - 1 ) ); + if ( dest < min ) { + DBGC ( image, "bzImage %s not enough space for initrds\n", + image->name ); return -ENOBUFS; } + bzimg->initrd = phys_to_virt ( dest ); + DBGC ( image, "bzImage %s loading initrds from %#08lx downwards\n", + image->name, max ); return 0; } @@ -451,65 +382,21 @@ static int bzimage_check_initrds ( struct image *image, */ static void bzimage_load_initrds ( struct image *image, struct bzimage_context *bzimg ) { - struct image *initrd; - struct image *highest = NULL; - struct image *other; - userptr_t top; - userptr_t dest; - size_t offset; size_t len; - /* Reshuffle initrds into desired order */ - initrd_reshuffle ( userptr_add ( bzimg->pm_kernel, bzimg->pm_sz ) ); - - /* Find highest initrd */ - for_each_image ( initrd ) { - if ( ( highest == NULL ) || - ( userptr_sub ( initrd->data, highest->data ) > 0 ) ) { - highest = initrd; - } - } - /* Do nothing if there are no initrds */ - if ( ! highest ) + if ( ! bzimg->initrd ) return; - /* Find highest usable address */ - top = userptr_add ( highest->data, bzimage_align ( highest->len ) ); - if ( user_to_phys ( top, -1 ) > bzimg->mem_limit ) { - top = phys_to_user ( ( bzimg->mem_limit + 1 ) & - ~( INITRD_ALIGN - 1 ) ); - } - DBGC ( image, "bzImage %p loading initrds from %#08lx downwards\n", - image, user_to_phys ( top, -1 ) ); - - /* Load initrds in order */ - for_each_image ( initrd ) { - - /* Calculate cumulative length of following - * initrds (including padding). - */ - offset = 0; - for_each_image ( other ) { - if ( other == initrd ) - offset = 0; - offset += bzimage_load_initrd ( image, other, UNULL ); - offset = bzimage_align ( offset ); - } - - /* Load initrd at this address */ - dest = userptr_add ( top, -offset ); - len = bzimage_load_initrd ( image, initrd, dest ); - - /* Record initrd location */ - if ( ! bzimg->ramdisk_image ) - bzimg->ramdisk_image = user_to_phys ( dest, 0 ); - bzimg->ramdisk_size = ( user_to_phys ( dest, len ) - - bzimg->ramdisk_image ); - } - DBGC ( image, "bzImage %p initrds at [%#08lx,%#08lx)\n", - image, bzimg->ramdisk_image, - ( bzimg->ramdisk_image + bzimg->ramdisk_size ) ); + /* Reshuffle initrds into desired order */ + initrd_reshuffle(); + + /* Load initrds */ + DBGC ( image, "bzImage %s initrds at [%#08lx,%#08lx)\n", + image->name, virt_to_phys ( bzimg->initrd ), + ( virt_to_phys ( bzimg->initrd ) + bzimg->initrd_size ) ); + len = initrd_load_all ( bzimg->initrd ); + assert ( len == bzimg->initrd_size ); } /** @@ -523,21 +410,20 @@ static int bzimage_exec ( struct image *image ) { int rc; /* Read and parse header from image */ - if ( ( rc = bzimage_parse_header ( image, &bzimg, - image->data ) ) != 0 ) + if ( ( rc = bzimage_parse_header ( image, &bzimg ) ) != 0 ) return rc; /* Prepare segments */ if ( ( rc = prep_segment ( bzimg.rm_kernel, bzimg.rm_filesz, bzimg.rm_memsz ) ) != 0 ) { - DBGC ( image, "bzImage %p could not prepare RM segment: %s\n", - image, strerror ( rc ) ); + DBGC ( image, "bzImage %s could not prepare RM segment: %s\n", + image->name, strerror ( rc ) ); return rc; } if ( ( rc = prep_segment ( bzimg.pm_kernel, bzimg.pm_sz, bzimg.pm_sz ) ) != 0 ) { - DBGC ( image, "bzImage %p could not prepare PM segment: %s\n", - image, strerror ( rc ) ); + DBGC ( image, "bzImage %s could not prepare PM segment: %s\n", + image->name, strerror ( rc ) ); return rc; } @@ -553,10 +439,9 @@ static int bzimage_exec ( struct image *image ) { unregister_image ( image_get ( image ) ); /* Load segments */ - memcpy_user ( bzimg.rm_kernel, 0, image->data, - 0, bzimg.rm_filesz ); - memcpy_user ( bzimg.pm_kernel, 0, image->data, - bzimg.rm_filesz, bzimg.pm_sz ); + memcpy ( bzimg.rm_kernel, image->data, bzimg.rm_filesz ); + memcpy ( bzimg.pm_kernel, ( image->data + bzimg.rm_filesz ), + bzimg.pm_sz ); /* Store command line */ bzimage_set_cmdline ( image, &bzimg ); @@ -570,10 +455,10 @@ static int bzimage_exec ( struct image *image ) { bzimage_load_initrds ( image, &bzimg ); /* Update kernel header */ - bzimage_update_header ( image, &bzimg, bzimg.rm_kernel ); + bzimage_update_header ( image, &bzimg ); - DBGC ( image, "bzImage %p jumping to RM kernel at %04x:0000 " - "(stack %04x:%04zx)\n", image, ( bzimg.rm_kernel_seg + 0x20 ), + DBGC ( image, "bzImage %s jumping to RM kernel at %04x:0000 (stack " + "%04x:%04zx)\n", image->name, ( bzimg.rm_kernel_seg + 0x20 ), bzimg.rm_kernel_seg, bzimg.rm_heap ); /* Jump to the kernel */ @@ -609,8 +494,7 @@ int bzimage_probe ( struct image *image ) { int rc; /* Read and parse header from image */ - if ( ( rc = bzimage_parse_header ( image, &bzimg, - image->data ) ) != 0 ) + if ( ( rc = bzimage_parse_header ( image, &bzimg ) ) != 0 ) return rc; return 0; diff --git a/src/arch/x86/image/com32.c b/src/arch/x86/image/com32.c index 6f0e66041..2b31fb2e5 100644 --- a/src/arch/x86/image/com32.c +++ b/src/arch/x86/image/com32.c @@ -39,7 +39,7 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include <ipxe/image.h> #include <ipxe/segment.h> #include <ipxe/init.h> -#include <ipxe/io.h> +#include <ipxe/memmap.h> #include <ipxe/console.h> /** @@ -49,8 +49,7 @@ FILE_LICENCE ( GPL2_OR_LATER ); * @ret rc Return status code */ static int com32_exec_loop ( struct image *image ) { - struct memory_map memmap; - unsigned int i; + struct memmap_region region; int state; uint32_t avail_mem_top; @@ -59,21 +58,12 @@ static int com32_exec_loop ( struct image *image ) { switch ( state ) { case 0: /* First time through; invoke COM32 program */ - /* Get memory map */ - get_memmap ( &memmap ); - /* Find end of block covering COM32 image loading area */ - for ( i = 0, avail_mem_top = 0 ; i < memmap.count ; i++ ) { - if ( (memmap.regions[i].start <= COM32_START_PHYS) && - (memmap.regions[i].end > COM32_START_PHYS + image->len) ) { - avail_mem_top = memmap.regions[i].end; - break; - } - } - - DBGC ( image, "COM32 %p: available memory top = 0x%x\n", - image, avail_mem_top ); - + memmap_describe ( COM32_START_PHYS, 1, ®ion ); + assert ( memmap_is_usable ( ®ion ) ); + avail_mem_top = ( COM32_START_PHYS + memmap_size ( ®ion ) ); + DBGC ( image, "COM32 %s: available memory top = 0x%x\n", + image->name, avail_mem_top ); assert ( avail_mem_top != 0 ); /* Hook COMBOOT API interrupts */ @@ -114,32 +104,32 @@ static int com32_exec_loop ( struct image *image ) { /* Restore registers */ "popal\n\t" ) : - : "r" ( avail_mem_top ), - "r" ( virt_to_phys ( com32_cfarcall_wrapper ) ), - "r" ( virt_to_phys ( com32_farcall_wrapper ) ), - "r" ( get_fbms() * 1024 - ( COM32_BOUNCE_SEG << 4 ) ), + : "R" ( avail_mem_top ), + "R" ( virt_to_phys ( com32_cfarcall_wrapper ) ), + "R" ( virt_to_phys ( com32_farcall_wrapper ) ), + "R" ( get_fbms() * 1024 - ( COM32_BOUNCE_SEG << 4 ) ), "i" ( COM32_BOUNCE_SEG << 4 ), - "r" ( virt_to_phys ( com32_intcall_wrapper ) ), - "r" ( virt_to_phys ( image->cmdline ? + "R" ( virt_to_phys ( com32_intcall_wrapper ) ), + "R" ( virt_to_phys ( image->cmdline ? image->cmdline : "" ) ), "i" ( COM32_START_PHYS ) : "memory" ); - DBGC ( image, "COM32 %p: returned\n", image ); + DBGC ( image, "COM32 %s: returned\n", image->name ); break; case COMBOOT_EXIT: - DBGC ( image, "COM32 %p: exited\n", image ); + DBGC ( image, "COM32 %s: exited\n", image->name ); break; case COMBOOT_EXIT_RUN_KERNEL: assert ( image->replacement ); - DBGC ( image, "COM32 %p: exited to run kernel %s\n", - image, image->replacement->name ); + DBGC ( image, "COM32 %s: exited to run kernel %s\n", + image->name, image->replacement->name ); break; case COMBOOT_EXIT_COMMAND: - DBGC ( image, "COM32 %p: exited after executing command\n", - image ); + DBGC ( image, "COM32 %s: exited after executing command\n", + image->name ); break; default: @@ -162,17 +152,15 @@ static int com32_exec_loop ( struct image *image ) { static int com32_identify ( struct image *image ) { const char *ext; static const uint8_t magic[] = { 0xB8, 0xFF, 0x4C, 0xCD, 0x21 }; - uint8_t buf[5]; - if ( image->len >= 5 ) { + if ( image->len >= sizeof ( magic ) ) { /* Check for magic number * mov eax,21cd4cffh * B8 FF 4C CD 21 */ - copy_from_user ( buf, image->data, 0, sizeof(buf) ); - if ( ! memcmp ( buf, magic, sizeof(buf) ) ) { - DBGC ( image, "COM32 %p: found magic number\n", - image ); + if ( memcmp ( image->data, magic, sizeof ( magic) ) == 0 ) { + DBGC ( image, "COM32 %s: found magic number\n", + image->name ); return 0; } } @@ -182,16 +170,16 @@ static int com32_identify ( struct image *image ) { ext = strrchr( image->name, '.' ); if ( ! ext ) { - DBGC ( image, "COM32 %p: no extension\n", - image ); + DBGC ( image, "COM32 %s: no extension\n", + image->name ); return -ENOEXEC; } ++ext; if ( strcasecmp( ext, "c32" ) ) { - DBGC ( image, "COM32 %p: unrecognized extension %s\n", - image, ext ); + DBGC ( image, "COM32 %s: unrecognized extension %s\n", + image->name, ext ); return -ENOEXEC; } @@ -206,20 +194,20 @@ static int com32_identify ( struct image *image ) { */ static int com32_load_image ( struct image *image ) { size_t filesz, memsz; - userptr_t buffer; + void *buffer; int rc; filesz = image->len; memsz = filesz; - buffer = phys_to_user ( COM32_START_PHYS ); + buffer = phys_to_virt ( COM32_START_PHYS ); if ( ( rc = prep_segment ( buffer, filesz, memsz ) ) != 0 ) { - DBGC ( image, "COM32 %p: could not prepare segment: %s\n", - image, strerror ( rc ) ); + DBGC ( image, "COM32 %s: could not prepare segment: %s\n", + image->name, strerror ( rc ) ); return rc; } /* Copy image to segment */ - memcpy_user ( buffer, 0, image->data, 0, filesz ); + memcpy ( buffer, image->data, filesz ); return 0; } @@ -230,22 +218,20 @@ static int com32_load_image ( struct image *image ) { * @ret rc Return status code */ static int com32_prepare_bounce_buffer ( struct image * image ) { - unsigned int seg; - userptr_t seg_userptr; + void *seg; size_t filesz, memsz; int rc; - seg = COM32_BOUNCE_SEG; - seg_userptr = real_to_user ( seg, 0 ); + seg = real_to_virt ( COM32_BOUNCE_SEG, 0 ); /* Ensure the entire 64k segment is free */ memsz = 0xFFFF; filesz = 0; /* Prepare, verify, and load the real-mode segment */ - if ( ( rc = prep_segment ( seg_userptr, filesz, memsz ) ) != 0 ) { - DBGC ( image, "COM32 %p: could not prepare bounce buffer segment: %s\n", - image, strerror ( rc ) ); + if ( ( rc = prep_segment ( seg, filesz, memsz ) ) != 0 ) { + DBGC ( image, "COM32 %s: could not prepare bounce buffer segment: %s\n", + image->name, strerror ( rc ) ); return rc; } @@ -261,8 +247,6 @@ static int com32_prepare_bounce_buffer ( struct image * image ) { static int com32_probe ( struct image *image ) { int rc; - DBGC ( image, "COM32 %p: name '%s'\n", image, image->name ); - /* Check if this is a COMBOOT image */ if ( ( rc = com32_identify ( image ) ) != 0 ) { return rc; diff --git a/src/arch/x86/image/comboot.c b/src/arch/x86/image/comboot.c index 9a847f0ff..6eba027c6 100644 --- a/src/arch/x86/image/comboot.c +++ b/src/arch/x86/image/comboot.c @@ -35,7 +35,6 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include <realmode.h> #include <basemem.h> #include <comboot.h> -#include <ipxe/uaccess.h> #include <ipxe/image.h> #include <ipxe/segment.h> #include <ipxe/init.h> @@ -67,62 +66,53 @@ struct comboot_psp { * * @v image COMBOOT image */ -static void comboot_copy_cmdline ( struct image * image, userptr_t seg_userptr ) { +static void comboot_copy_cmdline ( struct image * image, void *seg ) { const char *cmdline = ( image->cmdline ? image->cmdline : "" ); int cmdline_len = strlen ( cmdline ); + uint8_t *psp_cmdline; + + /* Limit length of command line */ if( cmdline_len > COMBOOT_MAX_CMDLINE_LEN ) cmdline_len = COMBOOT_MAX_CMDLINE_LEN; - uint8_t len_byte = cmdline_len; - char spc = ' ', cr = '\r'; /* Copy length to byte before command line */ - copy_to_user ( seg_userptr, COMBOOT_PSP_CMDLINE_OFFSET - 1, - &len_byte, 1 ); + psp_cmdline = ( seg + COMBOOT_PSP_CMDLINE_OFFSET ); + psp_cmdline[-1] = cmdline_len; /* Command line starts with space */ - copy_to_user ( seg_userptr, - COMBOOT_PSP_CMDLINE_OFFSET, - &spc, 1 ); + psp_cmdline[0] = ' '; /* Copy command line */ - copy_to_user ( seg_userptr, - COMBOOT_PSP_CMDLINE_OFFSET + 1, - cmdline, cmdline_len ); + memcpy ( &psp_cmdline[1], cmdline, cmdline_len ); /* Command line ends with CR */ - copy_to_user ( seg_userptr, - COMBOOT_PSP_CMDLINE_OFFSET + cmdline_len + 1, - &cr, 1 ); + psp_cmdline[ 1 + cmdline_len ] = '\r'; } /** * Initialize PSP * * @v image COMBOOT image - * @v seg_userptr segment to initialize + * @v seg segment to initialize */ -static void comboot_init_psp ( struct image * image, userptr_t seg_userptr ) { - struct comboot_psp psp; +static void comboot_init_psp ( struct image * image, void *seg ) { + struct comboot_psp *psp; /* Fill PSP */ + psp = seg; /* INT 20h instruction, byte order reversed */ - psp.int20 = 0x20CD; + psp->int20 = 0x20CD; /* get_fbms() returns BIOS free base memory counter, which is in * kilobytes; x * 1024 / 16 == x * 64 == x << 6 */ - psp.first_non_free_para = get_fbms() << 6; - - DBGC ( image, "COMBOOT %p: first non-free paragraph = 0x%x\n", - image, psp.first_non_free_para ); + psp->first_non_free_para = get_fbms() << 6; - /* Copy the PSP to offset 0 of segment. - * The rest of the PSP was already zeroed by - * comboot_prepare_segment. */ - copy_to_user ( seg_userptr, 0, &psp, sizeof( psp ) ); + DBGC ( image, "COMBOOT %s: first non-free paragraph = 0x%x\n", + image->name, psp->first_non_free_para ); /* Copy the command line to the PSP */ - comboot_copy_cmdline ( image, seg_userptr ); + comboot_copy_cmdline ( image, seg ); } /** @@ -132,7 +122,7 @@ static void comboot_init_psp ( struct image * image, userptr_t seg_userptr ) { * @ret rc Return status code */ static int comboot_exec_loop ( struct image *image ) { - userptr_t seg_userptr = real_to_user ( COMBOOT_PSP_SEG, 0 ); + void *seg = real_to_virt ( COMBOOT_PSP_SEG, 0 ); int state; state = rmsetjmp ( comboot_return ); @@ -141,7 +131,7 @@ static int comboot_exec_loop ( struct image *image ) { case 0: /* First time through; invoke COMBOOT program */ /* Initialize PSP */ - comboot_init_psp ( image, seg_userptr ); + comboot_init_psp ( image, seg ); /* Hook COMBOOT API interrupts */ hook_comboot_interrupts(); @@ -181,23 +171,23 @@ static int comboot_exec_loop ( struct image *image ) { "xorw %%di, %%di\n\t" "xorw %%bp, %%bp\n\t" "lret\n\t" ) - : : "r" ( COMBOOT_PSP_SEG ) : "eax" ); - DBGC ( image, "COMBOOT %p: returned\n", image ); + : : "R" ( COMBOOT_PSP_SEG ) : "eax" ); + DBGC ( image, "COMBOOT %s: returned\n", image->name ); break; case COMBOOT_EXIT: - DBGC ( image, "COMBOOT %p: exited\n", image ); + DBGC ( image, "COMBOOT %s: exited\n", image->name ); break; case COMBOOT_EXIT_RUN_KERNEL: assert ( image->replacement ); - DBGC ( image, "COMBOOT %p: exited to run kernel %s\n", - image, image->replacement->name ); + DBGC ( image, "COMBOOT %s: exited to run kernel %s\n", + image->name, image->replacement->name ); break; case COMBOOT_EXIT_COMMAND: - DBGC ( image, "COMBOOT %p: exited after executing command\n", - image ); + DBGC ( image, "COMBOOT %s: exited after executing command\n", + image->name ); break; default: @@ -223,16 +213,16 @@ static int comboot_identify ( struct image *image ) { ext = strrchr( image->name, '.' ); if ( ! ext ) { - DBGC ( image, "COMBOOT %p: no extension\n", - image ); + DBGC ( image, "COMBOOT %s: no extension\n", + image->name ); return -ENOEXEC; } ++ext; if ( strcasecmp( ext, "cbt" ) ) { - DBGC ( image, "COMBOOT %p: unrecognized extension %s\n", - image, ext ); + DBGC ( image, "COMBOOT %s: unrecognized extension %s\n", + image->name, ext ); return -ENOEXEC; } @@ -246,12 +236,12 @@ static int comboot_identify ( struct image *image ) { */ static int comboot_prepare_segment ( struct image *image ) { - userptr_t seg_userptr; + void *seg; size_t filesz, memsz; int rc; /* Load image in segment */ - seg_userptr = real_to_user ( COMBOOT_PSP_SEG, 0 ); + seg = real_to_virt ( COMBOOT_PSP_SEG, 0 ); /* Allow etra 0x100 bytes before image for PSP */ filesz = image->len + 0x100; @@ -260,17 +250,17 @@ static int comboot_prepare_segment ( struct image *image ) memsz = 0xFFFF; /* Prepare, verify, and load the real-mode segment */ - if ( ( rc = prep_segment ( seg_userptr, filesz, memsz ) ) != 0 ) { - DBGC ( image, "COMBOOT %p: could not prepare segment: %s\n", - image, strerror ( rc ) ); + if ( ( rc = prep_segment ( seg, filesz, memsz ) ) != 0 ) { + DBGC ( image, "COMBOOT %s: could not prepare segment: %s\n", + image->name, strerror ( rc ) ); return rc; } /* Zero PSP */ - memset_user ( seg_userptr, 0, 0, 0x100 ); + memset ( seg, 0, 0x100 ); /* Copy image to segment:0100 */ - memcpy_user ( seg_userptr, 0x100, image->data, 0, image->len ); + memcpy ( ( seg + 0x100 ), image->data, image->len ); return 0; } @@ -284,9 +274,6 @@ static int comboot_prepare_segment ( struct image *image ) static int comboot_probe ( struct image *image ) { int rc; - DBGC ( image, "COMBOOT %p: name '%s'\n", - image, image->name ); - /* Check if this is a COMBOOT image */ if ( ( rc = comboot_identify ( image ) ) != 0 ) { @@ -307,8 +294,8 @@ static int comboot_exec ( struct image *image ) { /* Sanity check for filesize */ if( image->len >= 0xFF00 ) { - DBGC( image, "COMBOOT %p: image too large\n", - image ); + DBGC( image, "COMBOOT %s: image too large\n", + image->name ); return -ENOEXEC; } diff --git a/src/arch/x86/image/elfboot.c b/src/arch/x86/image/elfboot.c index dc3568929..d0f91d1c0 100644 --- a/src/arch/x86/image/elfboot.c +++ b/src/arch/x86/image/elfboot.c @@ -23,8 +23,10 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +#include <string.h> #include <errno.h> #include <elf.h> +#include <librm.h> #include <ipxe/image.h> #include <ipxe/elf.h> #include <ipxe/features.h> @@ -52,8 +54,8 @@ static int elfboot_exec ( struct image *image ) { /* Load the image using core ELF support */ if ( ( rc = elf_load ( image, &entry, &max ) ) != 0 ) { - DBGC ( image, "ELF %p could not load: %s\n", - image, strerror ( rc ) ); + DBGC ( image, "ELF %s could not load: %s\n", + image->name, strerror ( rc ) ); return rc; } @@ -63,14 +65,15 @@ static int elfboot_exec ( struct image *image ) { shutdown_boot(); /* Jump to OS with flat physical addressing */ - DBGC ( image, "ELF %p starting execution at %lx\n", image, entry ); + DBGC ( image, "ELF %s starting execution at %lx\n", + image->name, entry ); __asm__ __volatile__ ( PHYS_CODE ( "pushl %%ebp\n\t" /* gcc bug */ "call *%%edi\n\t" "popl %%ebp\n\t" /* gcc bug */ ) : : "D" ( entry ) : "eax", "ebx", "ecx", "edx", "esi", "memory" ); - DBGC ( image, "ELF %p returned\n", image ); + DBGC ( image, "ELF %s returned\n", image->name ); /* It isn't safe to continue after calling shutdown() */ while ( 1 ) {} @@ -86,13 +89,13 @@ static int elfboot_exec ( struct image *image ) { * @v dest Destination address * @ret rc Return status code */ -static int elfboot_check_segment ( struct image *image, Elf_Phdr *phdr, +static int elfboot_check_segment ( struct image *image, const Elf_Phdr *phdr, physaddr_t dest ) { /* Check that ELF segment uses flat physical addressing */ if ( phdr->p_vaddr != dest ) { - DBGC ( image, "ELF %p uses virtual addressing (phys %x, " - "virt %x)\n", image, phdr->p_paddr, phdr->p_vaddr ); + DBGC ( image, "ELF %s uses virtual addressing (phys %x, virt " + "%x)\n", image->name, phdr->p_paddr, phdr->p_vaddr ); return -ENOEXEC; } @@ -106,7 +109,7 @@ static int elfboot_check_segment ( struct image *image, Elf_Phdr *phdr, * @ret rc Return status code */ static int elfboot_probe ( struct image *image ) { - Elf32_Ehdr ehdr; + const Elf32_Ehdr *ehdr; static const uint8_t e_ident[] = { [EI_MAG0] = ELFMAG0, [EI_MAG1] = ELFMAG1, @@ -121,16 +124,22 @@ static int elfboot_probe ( struct image *image ) { int rc; /* Read ELF header */ - copy_from_user ( &ehdr, image->data, 0, sizeof ( ehdr ) ); - if ( memcmp ( ehdr.e_ident, e_ident, sizeof ( e_ident ) ) != 0 ) { - DBGC ( image, "Invalid ELF identifier\n" ); + if ( image->len < sizeof ( *ehdr ) ) { + DBGC ( image, "ELF %s too short for ELF header\n", + image->name ); + return -ENOEXEC; + } + ehdr = image->data; + if ( memcmp ( ehdr->e_ident, e_ident, sizeof ( e_ident ) ) != 0 ) { + DBGC ( image, "ELF %s invalid identifier\n", image->name ); return -ENOEXEC; } /* Check that this image uses flat physical addressing */ - if ( ( rc = elf_segments ( image, &ehdr, elfboot_check_segment, + if ( ( rc = elf_segments ( image, ehdr, elfboot_check_segment, &entry, &max ) ) != 0 ) { - DBGC ( image, "Unloadable ELF image\n" ); + DBGC ( image, "ELF %s is not loadable: %s\n", + image->name, strerror ( rc ) ); return rc; } diff --git a/src/arch/x86/image/initrd.c b/src/arch/x86/image/initrd.c deleted file mode 100644 index d7b1f5773..000000000 --- a/src/arch/x86/image/initrd.c +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright (C) 2012 Michael Brown <mbrown@fensystems.co.uk>. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - * - * You can also choose to distribute this program under the terms of - * the Unmodified Binary Distribution Licence (as given in the file - * COPYING.UBDL), provided that you have satisfied its requirements. - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <errno.h> -#include <initrd.h> -#include <ipxe/image.h> -#include <ipxe/uaccess.h> -#include <ipxe/init.h> -#include <ipxe/memblock.h> -#include <ipxe/cpio.h> - -/** @file - * - * Initial ramdisk (initrd) reshuffling - * - */ - -/** Maximum address available for initrd */ -userptr_t initrd_top; - -/** Minimum address available for initrd */ -userptr_t initrd_bottom; - -/** - * Squash initrds as high as possible in memory - * - * @v top Highest possible address - * @ret used Lowest address used by initrds - */ -static userptr_t initrd_squash_high ( userptr_t top ) { - userptr_t current = top; - struct image *initrd; - struct image *highest; - size_t len; - - /* Squash up any initrds already within or below the region */ - while ( 1 ) { - - /* Find the highest image not yet in its final position */ - highest = NULL; - for_each_image ( initrd ) { - if ( ( userptr_sub ( initrd->data, current ) < 0 ) && - ( ( highest == NULL ) || - ( userptr_sub ( initrd->data, - highest->data ) > 0 ) ) ) { - highest = initrd; - } - } - if ( ! highest ) - break; - - /* Move this image to its final position */ - len = ( ( highest->len + INITRD_ALIGN - 1 ) & - ~( INITRD_ALIGN - 1 ) ); - current = userptr_sub ( current, len ); - DBGC ( &images, "INITRD squashing %s [%#08lx,%#08lx)->" - "[%#08lx,%#08lx)\n", highest->name, - user_to_phys ( highest->data, 0 ), - user_to_phys ( highest->data, highest->len ), - user_to_phys ( current, 0 ), - user_to_phys ( current, highest->len ) ); - memmove_user ( current, 0, highest->data, 0, highest->len ); - highest->data = current; - } - - /* Copy any remaining initrds (e.g. embedded images) to the region */ - for_each_image ( initrd ) { - if ( userptr_sub ( initrd->data, top ) >= 0 ) { - len = ( ( initrd->len + INITRD_ALIGN - 1 ) & - ~( INITRD_ALIGN - 1 ) ); - current = userptr_sub ( current, len ); - DBGC ( &images, "INITRD copying %s [%#08lx,%#08lx)->" - "[%#08lx,%#08lx)\n", initrd->name, - user_to_phys ( initrd->data, 0 ), - user_to_phys ( initrd->data, initrd->len ), - user_to_phys ( current, 0 ), - user_to_phys ( current, initrd->len ) ); - memcpy_user ( current, 0, initrd->data, 0, - initrd->len ); - initrd->data = current; - } - } - - return current; -} - -/** - * Swap position of two adjacent initrds - * - * @v low Lower initrd - * @v high Higher initrd - * @v free Free space - * @v free_len Length of free space - */ -static void initrd_swap ( struct image *low, struct image *high, - userptr_t free, size_t free_len ) { - size_t len = 0; - size_t frag_len; - size_t new_len; - - DBGC ( &images, "INITRD swapping %s [%#08lx,%#08lx)<->[%#08lx,%#08lx) " - "%s\n", low->name, user_to_phys ( low->data, 0 ), - user_to_phys ( low->data, low->len ), - user_to_phys ( high->data, 0 ), - user_to_phys ( high->data, high->len ), high->name ); - - /* Round down length of free space */ - free_len &= ~( INITRD_ALIGN - 1 ); - assert ( free_len > 0 ); - - /* Swap image data */ - while ( len < high->len ) { - - /* Calculate maximum fragment length */ - frag_len = ( high->len - len ); - if ( frag_len > free_len ) - frag_len = free_len; - new_len = ( ( len + frag_len + INITRD_ALIGN - 1 ) & - ~( INITRD_ALIGN - 1 ) ); - - /* Swap fragments */ - memcpy_user ( free, 0, high->data, len, frag_len ); - memmove_user ( low->data, new_len, low->data, len, low->len ); - memcpy_user ( low->data, len, free, 0, frag_len ); - len = new_len; - } - - /* Adjust data pointers */ - high->data = low->data; - low->data = userptr_add ( low->data, len ); -} - -/** - * Swap position of any two adjacent initrds not currently in the correct order - * - * @v free Free space - * @v free_len Length of free space - * @ret swapped A pair of initrds was swapped - */ -static int initrd_swap_any ( userptr_t free, size_t free_len ) { - struct image *low; - struct image *high; - size_t padded_len; - userptr_t adjacent; - - /* Find any pair of initrds that can be swapped */ - for_each_image ( low ) { - - /* Calculate location of adjacent image (if any) */ - padded_len = ( ( low->len + INITRD_ALIGN - 1 ) & - ~( INITRD_ALIGN - 1 ) ); - adjacent = userptr_add ( low->data, padded_len ); - - /* Search for adjacent image */ - for_each_image ( high ) { - - /* Stop search if all remaining potential - * adjacent images are already in the correct - * order. - */ - if ( high == low ) - break; - - /* If we have found the adjacent image, swap and exit */ - if ( high->data == adjacent ) { - initrd_swap ( low, high, free, free_len ); - return 1; - } - } - } - - /* Nothing swapped */ - return 0; -} - -/** - * Dump initrd locations (for debug) - * - */ -static void initrd_dump ( void ) { - struct image *initrd; - - /* Do nothing unless debugging is enabled */ - if ( ! DBG_LOG ) - return; - - /* Dump initrd locations */ - for_each_image ( initrd ) { - DBGC ( &images, "INITRD %s at [%#08lx,%#08lx)\n", - initrd->name, user_to_phys ( initrd->data, 0 ), - user_to_phys ( initrd->data, initrd->len ) ); - DBGC2_MD5A ( &images, user_to_phys ( initrd->data, 0 ), - user_to_virt ( initrd->data, 0 ), initrd->len ); - } -} - -/** - * Reshuffle initrds into desired order at top of memory - * - * @v bottom Lowest address available for initrds - * - * After this function returns, the initrds have been rearranged in - * memory and the external heap structures will have been corrupted. - * Reshuffling must therefore take place immediately prior to jumping - * to the loaded OS kernel; no further execution within iPXE is - * permitted. - */ -void initrd_reshuffle ( userptr_t bottom ) { - userptr_t top; - userptr_t used; - userptr_t free; - size_t free_len; - - /* Calculate limits of available space for initrds */ - top = initrd_top; - if ( userptr_sub ( initrd_bottom, bottom ) > 0 ) - bottom = initrd_bottom; - - /* Debug */ - DBGC ( &images, "INITRD region [%#08lx,%#08lx)\n", - user_to_phys ( bottom, 0 ), user_to_phys ( top, 0 ) ); - initrd_dump(); - - /* Squash initrds as high as possible in memory */ - used = initrd_squash_high ( top ); - - /* Calculate available free space */ - free = bottom; - free_len = userptr_sub ( used, free ); - - /* Bubble-sort initrds into desired order */ - while ( initrd_swap_any ( free, free_len ) ) {} - - /* Debug */ - initrd_dump(); -} - -/** - * Check that there is enough space to reshuffle initrds - * - * @v len Total length of initrds (including padding) - * @v bottom Lowest address available for initrds - * @ret rc Return status code - */ -int initrd_reshuffle_check ( size_t len, userptr_t bottom ) { - userptr_t top; - size_t available; - - /* Calculate limits of available space for initrds */ - top = initrd_top; - if ( userptr_sub ( initrd_bottom, bottom ) > 0 ) - bottom = initrd_bottom; - available = userptr_sub ( top, bottom ); - - /* Allow for a sensible minimum amount of free space */ - len += INITRD_MIN_FREE_LEN; - - /* Check for available space */ - return ( ( len < available ) ? 0 : -ENOBUFS ); -} - -/** - * initrd startup function - * - */ -static void initrd_startup ( void ) { - size_t len; - - /* Record largest memory block available. Do this after any - * allocations made during driver startup (e.g. large host - * memory blocks for Infiniband devices, which may still be in - * use at the time of rearranging if a SAN device is hooked) - * but before any allocations for downloaded images (which we - * can safely reuse when rearranging). - */ - len = largest_memblock ( &initrd_bottom ); - initrd_top = userptr_add ( initrd_bottom, len ); -} - -/** initrd startup function */ -struct startup_fn startup_initrd __startup_fn ( STARTUP_LATE ) = { - .name = "initrd", - .startup = initrd_startup, -}; diff --git a/src/arch/x86/image/multiboot.c b/src/arch/x86/image/multiboot.c index cada021ab..40d6941da 100644 --- a/src/arch/x86/image/multiboot.c +++ b/src/arch/x86/image/multiboot.c @@ -31,14 +31,14 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); */ #include <stdio.h> +#include <string.h> #include <errno.h> #include <assert.h> #include <realmode.h> #include <multiboot.h> -#include <ipxe/uaccess.h> #include <ipxe/image.h> #include <ipxe/segment.h> -#include <ipxe/io.h> +#include <ipxe/memmap.h> #include <ipxe/elf.h> #include <ipxe/init.h> #include <ipxe/features.h> @@ -59,6 +59,9 @@ FEATURE ( FEATURE_IMAGE, "MBOOT", DHCP_EB_FEATURE_MULTIBOOT, 1 ); */ #define MAX_MODULES 8 +/** Maximum number of memory map entries */ +#define MAX_MEMMAP 8 + /** * Maximum combined length of command lines * @@ -87,14 +90,6 @@ FEATURE ( FEATURE_IMAGE, "MBOOT", DHCP_EB_FEATURE_MULTIBOOT, 1 ); */ #define MB_UNSUPPORTED_FLAGS ( MB_COMPULSORY_FLAGS & ~MB_SUPPORTED_FLAGS ) -/** A multiboot header descriptor */ -struct multiboot_header_info { - /** The actual multiboot header */ - struct multiboot_header mb; - /** Offset of header within the multiboot image */ - size_t offset; -}; - /** Multiboot module command lines */ static char __bss16_array ( mb_cmdlines, [MB_MAX_CMDLINE] ); #define mb_cmdlines __use_data16 ( mb_cmdlines ) @@ -114,32 +109,43 @@ static void multiboot_build_memmap ( struct image *image, struct multiboot_info *mbinfo, struct multiboot_memory_map *mbmemmap, unsigned int limit ) { - struct memory_map memmap; - unsigned int i; - - /* Get memory map */ - get_memmap ( &memmap ); + struct memmap_region region; + unsigned int remaining; /* Translate into multiboot format */ memset ( mbmemmap, 0, sizeof ( *mbmemmap ) ); - for ( i = 0 ; i < memmap.count ; i++ ) { - if ( i >= limit ) { - DBGC ( image, "MULTIBOOT %p limit of %d memmap " - "entries reached\n", image, limit ); + remaining = limit; + for_each_memmap ( ®ion, 0 ) { + + /* Ignore any non-memory regions */ + if ( ! ( region.flags & MEMMAP_FL_MEMORY ) ) + continue; + DBGC_MEMMAP ( image, ®ion ); + + /* Check Multiboot memory map limit */ + if ( ! remaining ) { + DBGC ( image, "MULTIBOOT %s limit of %d memmap " + "entries reached\n", image->name, limit ); break; } - mbmemmap[i].size = ( sizeof ( mbmemmap[i] ) - - sizeof ( mbmemmap[i].size ) ); - mbmemmap[i].base_addr = memmap.regions[i].start; - mbmemmap[i].length = ( memmap.regions[i].end - - memmap.regions[i].start ); - mbmemmap[i].type = MBMEM_RAM; - mbinfo->mmap_length += sizeof ( mbmemmap[i] ); - if ( memmap.regions[i].start == 0 ) - mbinfo->mem_lower = ( memmap.regions[i].end / 1024 ); - if ( memmap.regions[i].start == 0x100000 ) - mbinfo->mem_upper = ( ( memmap.regions[i].end - - 0x100000 ) / 1024 ); + + /* Populate Multiboot memory map entry */ + mbmemmap->size = ( sizeof ( *mbmemmap ) - + sizeof ( mbmemmap->size ) ); + mbmemmap->base_addr = region.min; + mbmemmap->length = memmap_size ( ®ion ); + mbmemmap->type = MBMEM_RAM; + + /* Update Multiboot information */ + mbinfo->mmap_length += sizeof ( *mbmemmap ); + if ( mbmemmap->base_addr == 0 ) + mbinfo->mem_lower = ( mbmemmap->length / 1024 ); + if ( mbmemmap->base_addr == 0x100000 ) + mbinfo->mem_upper = ( mbmemmap->length / 1024 ); + + /* Move to next Multiboot memory map entry */ + mbmemmap++; + remaining--; } } @@ -199,8 +205,8 @@ static int multiboot_add_modules ( struct image *image, physaddr_t start, for_each_image ( module_image ) { if ( mbinfo->mods_count >= limit ) { - DBGC ( image, "MULTIBOOT %p limit of %d modules " - "reached\n", image, limit ); + DBGC ( image, "MULTIBOOT %s limit of %d modules " + "reached\n", image->name, limit ); break; } @@ -212,18 +218,18 @@ static int multiboot_add_modules ( struct image *image, physaddr_t start, start = ( ( start + 0xfff ) & ~0xfff ); /* Prepare segment */ - if ( ( rc = prep_segment ( phys_to_user ( start ), + if ( ( rc = prep_segment ( phys_to_virt ( start ), module_image->len, module_image->len ) ) != 0 ) { - DBGC ( image, "MULTIBOOT %p could not prepare module " - "%s: %s\n", image, module_image->name, + DBGC ( image, "MULTIBOOT %s could not prepare module " + "%s: %s\n", image->name, module_image->name, strerror ( rc ) ); return rc; } /* Copy module */ - memcpy_user ( phys_to_user ( start ), 0, - module_image->data, 0, module_image->len ); + memcpy ( phys_to_virt ( start ), module_image->data, + module_image->len ); /* Add module to list */ module = &modules[mbinfo->mods_count++]; @@ -231,8 +237,8 @@ static int multiboot_add_modules ( struct image *image, physaddr_t start, module->mod_end = ( start + module_image->len ); module->string = multiboot_add_cmdline ( module_image ); module->reserved = 0; - DBGC ( image, "MULTIBOOT %p module %s is [%x,%x)\n", - image, module_image->name, module->mod_start, + DBGC ( image, "MULTIBOOT %s module %s is [%x,%x)\n", + image->name, module_image->name, module->mod_start, module->mod_end ); start += module_image->len; } @@ -255,8 +261,7 @@ static char __bss16_array ( mb_bootloader_name, [32] ); #define mb_bootloader_name __use_data16 ( mb_bootloader_name ) /** The multiboot memory map */ -static struct multiboot_memory_map - __bss16_array ( mbmemmap, [MAX_MEMORY_REGIONS] ); +static struct multiboot_memory_map __bss16_array ( mbmemmap, [MAX_MEMMAP] ); #define mbmemmap __use_data16 ( mbmemmap ) /** The multiboot module list */ @@ -267,94 +272,101 @@ static struct multiboot_module __bss16_array ( mbmodules, [MAX_MODULES] ); * Find multiboot header * * @v image Multiboot file - * @v hdr Multiboot header descriptor to fill in - * @ret rc Return status code + * @ret offset Offset to Multiboot header, or negative error */ -static int multiboot_find_header ( struct image *image, - struct multiboot_header_info *hdr ) { - uint32_t buf[64]; +static int multiboot_find_header ( struct image *image ) { + const struct multiboot_header *mb; size_t offset; - unsigned int buf_idx; uint32_t checksum; - /* Scan through first 8kB of image file 256 bytes at a time. - * (Use the buffering to avoid the overhead of a - * copy_from_user() for every dword.) - */ - for ( offset = 0 ; offset < 8192 ; offset += sizeof ( buf[0] ) ) { + /* Scan through first 8kB of image file */ + for ( offset = 0 ; offset < 8192 ; offset += 4 ) { /* Check for end of image */ - if ( offset > image->len ) + if ( ( offset + sizeof ( *mb ) ) > image->len ) break; - /* Refill buffer if applicable */ - buf_idx = ( ( offset % sizeof ( buf ) ) / sizeof ( buf[0] ) ); - if ( buf_idx == 0 ) { - copy_from_user ( buf, image->data, offset, - sizeof ( buf ) ); - } + mb = ( image->data + offset ); /* Check signature */ - if ( buf[buf_idx] != MULTIBOOT_HEADER_MAGIC ) + if ( mb->magic != MULTIBOOT_HEADER_MAGIC ) continue; /* Copy header and verify checksum */ - copy_from_user ( &hdr->mb, image->data, offset, - sizeof ( hdr->mb ) ); - checksum = ( hdr->mb.magic + hdr->mb.flags + - hdr->mb.checksum ); + checksum = ( mb->magic + mb->flags + mb->checksum ); if ( checksum != 0 ) continue; - /* Record offset of multiboot header and return */ - hdr->offset = offset; - return 0; + /* Return header */ + return offset; } /* No multiboot header found */ + DBGC ( image, "MULTIBOOT %s has no multiboot header\n", + image->name ); return -ENOEXEC; } /** * Load raw multiboot image into memory * - * @v image Multiboot file - * @v hdr Multiboot header descriptor + * @v image Multiboot image + * @v offset Offset to Multiboot header * @ret entry Entry point * @ret max Maximum used address * @ret rc Return status code */ -static int multiboot_load_raw ( struct image *image, - struct multiboot_header_info *hdr, +static int multiboot_load_raw ( struct image *image, size_t offset, physaddr_t *entry, physaddr_t *max ) { - size_t offset; + const struct multiboot_header *mb = ( image->data + offset ); size_t filesz; size_t memsz; - userptr_t buffer; + void *buffer; int rc; /* Sanity check */ - if ( ! ( hdr->mb.flags & MB_FLAG_RAW ) ) { - DBGC ( image, "MULTIBOOT %p is not flagged as a raw image\n", - image ); + if ( ! ( mb->flags & MB_FLAG_RAW ) ) { + DBGC ( image, "MULTIBOOT %s is not flagged as a raw image\n", + image->name ); return -EINVAL; } - /* Verify and prepare segment */ - offset = ( hdr->offset - hdr->mb.header_addr + hdr->mb.load_addr ); - filesz = ( hdr->mb.load_end_addr ? - ( hdr->mb.load_end_addr - hdr->mb.load_addr ) : + /* Calculate starting offset within file */ + if ( ( mb->load_addr > mb->header_addr ) || + ( ( mb->header_addr - mb->load_addr ) > offset ) ) { + DBGC ( image, "MULTIBOOT %s has misplaced header\n", + image->name ); + return -EINVAL; + } + offset -= ( mb->header_addr - mb->load_addr ); + assert ( offset < image->len ); + + /* Calculate length of initialized data */ + filesz = ( mb->load_end_addr ? + ( mb->load_end_addr - mb->load_addr ) : ( image->len - offset ) ); - memsz = ( hdr->mb.bss_end_addr ? - ( hdr->mb.bss_end_addr - hdr->mb.load_addr ) : filesz ); - buffer = phys_to_user ( hdr->mb.load_addr ); + if ( filesz > image->len ) { + DBGC ( image, "MULTIBOOT %s has overlength data\n", + image->name ); + return -EINVAL; + } + + /* Calculate length of uninitialised data */ + memsz = ( mb->bss_end_addr ? + ( mb->bss_end_addr - mb->load_addr ) : filesz ); + DBGC ( image, "MULTIBOOT %s loading [%zx,%zx) to [%x,%zx,%zx)\n", + image->name, offset, ( offset + filesz ), mb->load_addr, + ( mb->load_addr + filesz ), ( mb->load_addr + memsz ) ); + + /* Verify and prepare segment */ + buffer = phys_to_virt ( mb->load_addr ); if ( ( rc = prep_segment ( buffer, filesz, memsz ) ) != 0 ) { - DBGC ( image, "MULTIBOOT %p could not prepare segment: %s\n", - image, strerror ( rc ) ); + DBGC ( image, "MULTIBOOT %s could not prepare segment: %s\n", + image->name, strerror ( rc ) ); return rc; } /* Copy image to segment */ - memcpy_user ( buffer, 0, image->data, offset, filesz ); + memcpy ( buffer, ( image->data + offset ), filesz ); /* Record execution entry point and maximum used address */ - *entry = hdr->mb.entry_addr; - *max = ( hdr->mb.load_addr + memsz ); + *entry = mb->entry_addr; + *max = ( mb->load_addr + memsz ); return 0; } @@ -373,8 +385,8 @@ static int multiboot_load_elf ( struct image *image, physaddr_t *entry, /* Load ELF image*/ if ( ( rc = elf_load ( image, entry, max ) ) != 0 ) { - DBGC ( image, "MULTIBOOT %p ELF image failed to load: %s\n", - image, strerror ( rc ) ); + DBGC ( image, "MULTIBOOT %s ELF image failed to load: %s\n", + image->name, strerror ( rc ) ); return rc; } @@ -388,22 +400,24 @@ static int multiboot_load_elf ( struct image *image, physaddr_t *entry, * @ret rc Return status code */ static int multiboot_exec ( struct image *image ) { - struct multiboot_header_info hdr; + const struct multiboot_header *mb; physaddr_t entry; physaddr_t max; + int offset; int rc; /* Locate multiboot header, if present */ - if ( ( rc = multiboot_find_header ( image, &hdr ) ) != 0 ) { - DBGC ( image, "MULTIBOOT %p has no multiboot header\n", - image ); + offset = multiboot_find_header ( image ); + if ( offset < 0 ) { + rc = offset; return rc; } + mb = ( image->data + offset ); /* Abort if we detect flags that we cannot support */ - if ( hdr.mb.flags & MB_UNSUPPORTED_FLAGS ) { - DBGC ( image, "MULTIBOOT %p flags %08x not supported\n", - image, ( hdr.mb.flags & MB_UNSUPPORTED_FLAGS ) ); + if ( mb->flags & MB_UNSUPPORTED_FLAGS ) { + DBGC ( image, "MULTIBOOT %s flags %#08x not supported\n", + image->name, ( mb->flags & MB_UNSUPPORTED_FLAGS ) ); return -ENOTSUP; } @@ -413,8 +427,10 @@ static int multiboot_exec ( struct image *image ) { * behaviour. */ if ( ( ( rc = multiboot_load_elf ( image, &entry, &max ) ) != 0 ) && - ( ( rc = multiboot_load_raw ( image, &hdr, &entry, &max ) ) != 0 )) + ( ( rc = multiboot_load_raw ( image, offset, &entry, + &max ) ) != 0 ) ) { return rc; + } /* Populate multiboot information structure */ memset ( &mbinfo, 0, sizeof ( mbinfo ) ); @@ -444,8 +460,8 @@ static int multiboot_exec ( struct image *image ) { ( sizeof(mbmemmap) / sizeof(mbmemmap[0]) ) ); /* Jump to OS with flat physical addressing */ - DBGC ( image, "MULTIBOOT %p starting execution at %lx\n", - image, entry ); + DBGC ( image, "MULTIBOOT %s starting execution at %lx\n", + image->name, entry ); __asm__ __volatile__ ( PHYS_CODE ( "pushl %%ebp\n\t" "call *%%edi\n\t" "popl %%ebp\n\t" ) @@ -454,7 +470,7 @@ static int multiboot_exec ( struct image *image ) { "D" ( entry ) : "ecx", "edx", "esi", "memory" ); - DBGC ( image, "MULTIBOOT %p returned\n", image ); + DBGC ( image, "MULTIBOOT %s returned\n", image->name ); /* It isn't safe to continue after calling shutdown() */ while ( 1 ) {} @@ -469,17 +485,19 @@ static int multiboot_exec ( struct image *image ) { * @ret rc Return status code */ static int multiboot_probe ( struct image *image ) { - struct multiboot_header_info hdr; + const struct multiboot_header *mb; + int offset; int rc; /* Locate multiboot header, if present */ - if ( ( rc = multiboot_find_header ( image, &hdr ) ) != 0 ) { - DBGC ( image, "MULTIBOOT %p has no multiboot header\n", - image ); + offset = multiboot_find_header ( image ); + if ( offset < 0 ) { + rc = offset; return rc; } - DBGC ( image, "MULTIBOOT %p found header with flags %08x\n", - image, hdr.mb.flags ); + mb = ( image->data + offset ); + DBGC ( image, "MULTIBOOT %s found header at +%#x with flags %#08x\n", + image->name, offset, mb->flags ); return 0; } diff --git a/src/arch/x86/image/nbi.c b/src/arch/x86/image/nbi.c index b691bee20..e0a46758e 100644 --- a/src/arch/x86/image/nbi.c +++ b/src/arch/x86/image/nbi.c @@ -1,3 +1,4 @@ +#include <string.h> #include <errno.h> #include <assert.h> #include <realmode.h> @@ -106,12 +107,12 @@ struct ebinfo { * @ret rc Return status code */ static int nbi_prepare_segment ( struct image *image, size_t offset __unused, - userptr_t dest, size_t filesz, size_t memsz ){ + void *dest, size_t filesz, size_t memsz ) { int rc; if ( ( rc = prep_segment ( dest, filesz, memsz ) ) != 0 ) { - DBGC ( image, "NBI %p could not prepare segment: %s\n", - image, strerror ( rc ) ); + DBGC ( image, "NBI %s could not prepare segment: %s\n", + image->name, strerror ( rc ) ); return rc; } @@ -129,9 +130,9 @@ static int nbi_prepare_segment ( struct image *image, size_t offset __unused, * @ret rc Return status code */ static int nbi_load_segment ( struct image *image, size_t offset, - userptr_t dest, size_t filesz, + void *dest, size_t filesz, size_t memsz __unused ) { - memcpy_user ( dest, 0, image->data, offset, filesz ); + memcpy ( dest, ( image->data + offset ), filesz ); return 0; } @@ -144,22 +145,22 @@ static int nbi_load_segment ( struct image *image, size_t offset, * @ret rc Return status code */ static int nbi_process_segments ( struct image *image, - struct imgheader *imgheader, + const struct imgheader *imgheader, int ( * process ) ( struct image *image, size_t offset, - userptr_t dest, + void *dest, size_t filesz, size_t memsz ) ) { - struct segheader sh; + const struct segheader *sh; size_t offset = 0; size_t sh_off; - userptr_t dest; + void *dest; size_t filesz; size_t memsz; int rc; /* Copy image header to target location */ - dest = real_to_user ( imgheader->location.segment, + dest = real_to_virt ( imgheader->location.segment, imgheader->location.offset ); filesz = memsz = NBI_HEADER_LENGTH; if ( ( rc = process ( image, offset, dest, filesz, memsz ) ) != 0 ) @@ -170,32 +171,32 @@ static int nbi_process_segments ( struct image *image, sh_off = NBI_LENGTH ( imgheader->length ); do { /* Read segment header */ - copy_from_user ( &sh, image->data, sh_off, sizeof ( sh ) ); - if ( sh.length == 0 ) { + sh = ( image->data + sh_off ); + if ( sh->length == 0 ) { /* Avoid infinite loop? */ - DBGC ( image, "NBI %p invalid segheader length 0\n", - image ); + DBGC ( image, "NBI %s invalid segheader length 0\n", + image->name ); return -ENOEXEC; } /* Calculate segment load address */ - switch ( NBI_LOADADDR_FLAGS ( sh.flags ) ) { + switch ( NBI_LOADADDR_FLAGS ( sh->flags ) ) { case NBI_LOADADDR_ABS: - dest = phys_to_user ( sh.loadaddr ); + dest = phys_to_virt ( sh->loadaddr ); break; case NBI_LOADADDR_AFTER: - dest = userptr_add ( dest, memsz + sh.loadaddr ); + dest = ( dest + memsz + sh->loadaddr ); break; case NBI_LOADADDR_BEFORE: - dest = userptr_add ( dest, -sh.loadaddr ); + dest = ( dest - sh->loadaddr ); break; case NBI_LOADADDR_END: /* Not correct according to the spec, but * maintains backwards compatibility with * previous versions of Etherboot. */ - dest = phys_to_user ( ( extmemsize() + 1024 ) * 1024 - - sh.loadaddr ); + dest = phys_to_virt ( ( extmemsize() + 1024 ) * 1024 + - sh->loadaddr ); break; default: /* Cannot be reached */ @@ -203,10 +204,11 @@ static int nbi_process_segments ( struct image *image, } /* Process this segment */ - filesz = sh.imglength; - memsz = sh.memlength; + filesz = sh->imglength; + memsz = sh->memlength; if ( ( offset + filesz ) > image->len ) { - DBGC ( image, "NBI %p segment outside file\n", image ); + DBGC ( image, "NBI %s segment outside file\n", + image->name ); return -ENOEXEC; } if ( ( rc = process ( image, offset, dest, @@ -216,17 +218,18 @@ static int nbi_process_segments ( struct image *image, offset += filesz; /* Next segheader */ - sh_off += NBI_LENGTH ( sh.length ); + sh_off += NBI_LENGTH ( sh->length ); if ( sh_off >= NBI_HEADER_LENGTH ) { - DBGC ( image, "NBI %p header overflow\n", image ); + DBGC ( image, "NBI %s header overflow\n", + image->name ); return -ENOEXEC; } - } while ( ! NBI_LAST_SEGHEADER ( sh.flags ) ); + } while ( ! NBI_LAST_SEGHEADER ( sh->flags ) ); if ( offset != image->len ) { - DBGC ( image, "NBI %p length wrong (file %zd, metadata %zd)\n", - image, image->len, offset ); + DBGC ( image, "NBI %s length wrong (file %zd, metadata %zd)\n", + image->name, image->len, offset ); return -ENOEXEC; } @@ -239,12 +242,13 @@ static int nbi_process_segments ( struct image *image, * @v imgheader Image header information * @ret rc Return status code, if image returns */ -static int nbi_boot16 ( struct image *image, struct imgheader *imgheader ) { +static int nbi_boot16 ( struct image *image, + const struct imgheader *imgheader ) { int discard_D, discard_S, discard_b; int32_t rc; - DBGC ( image, "NBI %p executing 16-bit image at %04x:%04x\n", image, - imgheader->execaddr.segoff.segment, + DBGC ( image, "NBI %s executing 16-bit image at %04x:%04x\n", + image->name, imgheader->execaddr.segoff.segment, imgheader->execaddr.segoff.offset ); __asm__ __volatile__ ( @@ -277,7 +281,8 @@ static int nbi_boot16 ( struct image *image, struct imgheader *imgheader ) { * @v imgheader Image header information * @ret rc Return status code, if image returns */ -static int nbi_boot32 ( struct image *image, struct imgheader *imgheader ) { +static int nbi_boot32 ( struct image *image, + const struct imgheader *imgheader ) { struct ebinfo loaderinfo = { product_major_version, product_minor_version, 0 @@ -285,8 +290,8 @@ static int nbi_boot32 ( struct image *image, struct imgheader *imgheader ) { int discard_D, discard_S, discard_b; int32_t rc; - DBGC ( image, "NBI %p executing 32-bit image at %lx\n", - image, imgheader->execaddr.linear ); + DBGC ( image, "NBI %s executing 32-bit image at %lx\n", + image->name, imgheader->execaddr.linear ); /* Jump to OS with flat physical addressing */ __asm__ __volatile__ ( @@ -321,14 +326,15 @@ static int nbi_prepare_dhcp ( struct image *image ) { boot_netdev = last_opened_netdev(); if ( ! boot_netdev ) { - DBGC ( image, "NBI %p could not identify a network device\n", - image ); + DBGC ( image, "NBI %s could not identify a network device\n", + image->name ); return -ENODEV; } if ( ( rc = create_fakedhcpack ( boot_netdev, basemem_packet, sizeof ( basemem_packet ) ) ) != 0 ) { - DBGC ( image, "NBI %p failed to build DHCP packet\n", image ); + DBGC ( image, "NBI %s failed to build DHCP packet\n", + image->name ); return rc; } @@ -342,15 +348,15 @@ static int nbi_prepare_dhcp ( struct image *image ) { * @ret rc Return status code */ static int nbi_exec ( struct image *image ) { - struct imgheader imgheader; + const struct imgheader *imgheader; int may_return; int rc; /* Retrieve image header */ - copy_from_user ( &imgheader, image->data, 0, sizeof ( imgheader ) ); + imgheader = image->data; - DBGC ( image, "NBI %p placing header at %hx:%hx\n", image, - imgheader.location.segment, imgheader.location.offset ); + DBGC ( image, "NBI %s placing header at %hx:%hx\n", image->name, + imgheader->location.segment, imgheader->location.offset ); /* NBI files can have overlaps between segments; the bss of * one segment may overlap the initialised data of another. I @@ -359,10 +365,10 @@ static int nbi_exec ( struct image *image ) { * passes: first to initialise the segments, then to copy the * data. This avoids zeroing out already-copied data. */ - if ( ( rc = nbi_process_segments ( image, &imgheader, + if ( ( rc = nbi_process_segments ( image, imgheader, nbi_prepare_segment ) ) != 0 ) return rc; - if ( ( rc = nbi_process_segments ( image, &imgheader, + if ( ( rc = nbi_process_segments ( image, imgheader, nbi_load_segment ) ) != 0 ) return rc; @@ -371,25 +377,25 @@ static int nbi_exec ( struct image *image ) { return rc; /* Shut down now if NBI image will not return */ - may_return = NBI_PROGRAM_RETURNS ( imgheader.flags ); + may_return = NBI_PROGRAM_RETURNS ( imgheader->flags ); if ( ! may_return ) shutdown_boot(); /* Execute NBI image */ - if ( NBI_LINEAR_EXEC_ADDR ( imgheader.flags ) ) { - rc = nbi_boot32 ( image, &imgheader ); + if ( NBI_LINEAR_EXEC_ADDR ( imgheader->flags ) ) { + rc = nbi_boot32 ( image, imgheader ); } else { - rc = nbi_boot16 ( image, &imgheader ); + rc = nbi_boot16 ( image, imgheader ); } if ( ! may_return ) { /* Cannot continue after shutdown() called */ - DBGC ( image, "NBI %p returned %d from non-returnable image\n", - image, rc ); + DBGC ( image, "NBI %s returned %d from non-returnable image\n", + image->name, rc ); while ( 1 ) {} } - DBGC ( image, "NBI %p returned %d\n", image, rc ); + DBGC ( image, "NBI %s returned %d\n", image->name, rc ); return rc; } @@ -401,18 +407,19 @@ static int nbi_exec ( struct image *image ) { * @ret rc Return status code */ static int nbi_probe ( struct image *image ) { - struct imgheader imgheader; + const struct imgheader *imgheader; /* If we don't have enough data give up */ if ( image->len < NBI_HEADER_LENGTH ) { - DBGC ( image, "NBI %p too short for an NBI image\n", image ); + DBGC ( image, "NBI %s too short for an NBI image\n", + image->name ); return -ENOEXEC; } + imgheader = image->data; /* Check image header */ - copy_from_user ( &imgheader, image->data, 0, sizeof ( imgheader ) ); - if ( imgheader.magic != NBI_MAGIC ) { - DBGC ( image, "NBI %p has no NBI signature\n", image ); + if ( imgheader->magic != NBI_MAGIC ) { + DBGC ( image, "NBI %s has no NBI signature\n", image->name ); return -ENOEXEC; } diff --git a/src/arch/x86/image/pxe_image.c b/src/arch/x86/image/pxe_image.c index b6bcb18b4..f88eadc09 100644 --- a/src/arch/x86/image/pxe_image.c +++ b/src/arch/x86/image/pxe_image.c @@ -30,10 +30,10 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * */ +#include <string.h> #include <pxe.h> #include <pxe_call.h> #include <pic8259.h> -#include <ipxe/uaccess.h> #include <ipxe/image.h> #include <ipxe/segment.h> #include <ipxe/netdevice.h> @@ -54,24 +54,24 @@ const char *pxe_cmdline; * @ret rc Return status code */ static int pxe_exec ( struct image *image ) { - userptr_t buffer = real_to_user ( 0, 0x7c00 ); + void *buffer = real_to_virt ( 0, 0x7c00 ); struct net_device *netdev; int rc; /* Verify and prepare segment */ if ( ( rc = prep_segment ( buffer, image->len, image->len ) ) != 0 ) { - DBGC ( image, "IMAGE %p could not prepare segment: %s\n", - image, strerror ( rc ) ); + DBGC ( image, "IMAGE %s could not prepare segment: %s\n", + image->name, strerror ( rc ) ); return rc; } /* Copy image to segment */ - memcpy_user ( buffer, 0, image->data, 0, image->len ); + memcpy ( buffer, image->data, image->len ); /* Arbitrarily pick the most recently opened network device */ if ( ( netdev = last_opened_netdev() ) == NULL ) { - DBGC ( image, "IMAGE %p could not locate PXE net device\n", - image ); + DBGC ( image, "IMAGE %s could not locate PXE net device\n", + image->name ); return -ENODEV; } netdev_get ( netdev ); @@ -142,7 +142,7 @@ int pxe_probe ( struct image *image ) { * @ret rc Return status code */ int pxe_probe_no_mz ( struct image *image ) { - uint16_t magic; + const uint16_t *magic; int rc; /* Probe PXE image */ @@ -152,11 +152,11 @@ int pxe_probe_no_mz ( struct image *image ) { /* Reject image with an "MZ" signature which may indicate an * EFI image incorrectly handed out to a BIOS system. */ - if ( image->len >= sizeof ( magic ) ) { - copy_from_user ( &magic, image->data, 0, sizeof ( magic ) ); - if ( magic == cpu_to_le16 ( EFI_IMAGE_DOS_SIGNATURE ) ) { - DBGC ( image, "IMAGE %p may be an EFI image\n", - image ); + if ( image->len >= sizeof ( *magic ) ) { + magic = image->data; + if ( *magic == cpu_to_le16 ( EFI_IMAGE_DOS_SIGNATURE ) ) { + DBGC ( image, "IMAGE %s may be an EFI image\n", + image->name ); return -ENOTTY; } } diff --git a/src/arch/x86/image/sdi.c b/src/arch/x86/image/sdi.c index fa2d0b73f..c0cded239 100644 --- a/src/arch/x86/image/sdi.c +++ b/src/arch/x86/image/sdi.c @@ -45,63 +45,36 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); FEATURE ( FEATURE_IMAGE, "SDI", DHCP_EB_FEATURE_SDI, 1 ); /** - * Parse SDI image header - * - * @v image SDI file - * @v sdi SDI header to fill in - * @ret rc Return status code - */ -static int sdi_parse_header ( struct image *image, struct sdi_header *sdi ) { - - /* Sanity check */ - if ( image->len < sizeof ( *sdi ) ) { - DBGC ( image, "SDI %p too short for SDI header\n", image ); - return -ENOEXEC; - } - - /* Read in header */ - copy_from_user ( sdi, image->data, 0, sizeof ( *sdi ) ); - - /* Check signature */ - if ( sdi->magic != SDI_MAGIC ) { - DBGC ( image, "SDI %p is not an SDI image\n", image ); - return -ENOEXEC; - } - - return 0; -} - -/** * Execute SDI image * * @v image SDI file * @ret rc Return status code */ static int sdi_exec ( struct image *image ) { - struct sdi_header sdi; + const struct sdi_header *sdi; uint32_t sdiptr; - int rc; - /* Parse image header */ - if ( ( rc = sdi_parse_header ( image, &sdi ) ) != 0 ) - return rc; + /* Sanity check */ + assert ( image->len >= sizeof ( *sdi ) ); + sdi = image->data; /* Check that image is bootable */ - if ( sdi.boot_size == 0 ) { - DBGC ( image, "SDI %p is not bootable\n", image ); + if ( sdi->boot_size == 0 ) { + DBGC ( image, "SDI %s is not bootable\n", image->name ); return -ENOTTY; } - DBGC ( image, "SDI %p image at %08lx+%08zx\n", - image, user_to_phys ( image->data, 0 ), image->len ); - DBGC ( image, "SDI %p boot code at %08lx+%llx\n", image, - user_to_phys ( image->data, sdi.boot_offset ), sdi.boot_size ); + DBGC ( image, "SDI %s image at %08lx+%08zx\n", + image->name, virt_to_phys ( image->data ), image->len ); + DBGC ( image, "SDI %s boot code at %08llx+%llx\n", image->name, + ( virt_to_phys ( image->data ) + sdi->boot_offset ), + sdi->boot_size ); /* Copy boot code */ - memcpy_user ( real_to_user ( SDI_BOOT_SEG, SDI_BOOT_OFF ), 0, - image->data, sdi.boot_offset, sdi.boot_size ); + memcpy ( real_to_virt ( SDI_BOOT_SEG, SDI_BOOT_OFF ), + ( image->data + sdi->boot_offset ), sdi->boot_size ); /* Jump to boot code */ - sdiptr = ( user_to_phys ( image->data, 0 ) | SDI_WTF ); + sdiptr = ( virt_to_phys ( image->data ) | SDI_WTF ); __asm__ __volatile__ ( REAL_CODE ( "ljmp %0, %1\n\t" ) : : "i" ( SDI_BOOT_SEG ), "i" ( SDI_BOOT_OFF ), @@ -122,12 +95,22 @@ static int sdi_exec ( struct image *image ) { * @ret rc Return status code */ static int sdi_probe ( struct image *image ) { - struct sdi_header sdi; - int rc; + const struct sdi_header *sdi; - /* Parse image */ - if ( ( rc = sdi_parse_header ( image, &sdi ) ) != 0 ) - return rc; + /* Sanity check */ + if ( image->len < sizeof ( *sdi ) ) { + DBGC ( image, "SDI %s too short for SDI header\n", + image->name ); + return -ENOEXEC; + } + sdi = image->data; + + /* Check signature */ + if ( sdi->magic != SDI_MAGIC ) { + DBGC ( image, "SDI %s is not an SDI image\n", + image->name ); + return -ENOEXEC; + } return 0; } diff --git a/src/arch/x86/image/ucode.c b/src/arch/x86/image/ucode.c index 499c0a940..fd4689e00 100644 --- a/src/arch/x86/image/ucode.c +++ b/src/arch/x86/image/ucode.c @@ -31,6 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdlib.h> #include <stdio.h> +#include <string.h> #include <assert.h> #include <errno.h> #include <ipxe/uaccess.h> @@ -149,41 +150,38 @@ static const char * ucode_vendor_name ( const union ucode_vendor_id *vendor ) { * * @v update Microcode update * @v control Microcode update control + * @v status Microcode update status * @v summary Microcode update summary * @v id APIC ID * @v optional Status report is optional * @ret rc Return status code */ -static int ucode_status ( struct ucode_update *update, - struct ucode_control *control, +static int ucode_status ( const struct ucode_update *update, + const struct ucode_control *control, + const struct ucode_status *status, struct ucode_summary *summary, unsigned int id, int optional ) { - struct ucode_status status; struct ucode_descriptor *desc; /* Sanity check */ assert ( id <= control->apic_max ); - /* Read status report */ - copy_from_user ( &status, phys_to_user ( control->status ), - ( id * sizeof ( status ) ), sizeof ( status ) ); - /* Ignore empty optional status reports */ - if ( optional && ( ! status.signature ) ) + if ( optional && ( ! status->signature ) ) return 0; DBGC ( update, "UCODE %#08x signature %#08x ucode %#08x->%#08x\n", - id, status.signature, status.before, status.after ); + id, status->signature, status->before, status->after ); /* Check CPU signature */ - if ( ! status.signature ) { + if ( ! status->signature ) { DBGC2 ( update, "UCODE %#08x has no signature\n", id ); return -ENOENT; } /* Check APIC ID is correct */ - if ( status.id != id ) { + if ( status->id != id ) { DBGC ( update, "UCODE %#08x wrong APIC ID %#08x\n", - id, status.id ); + id, status->id ); return -EINVAL; } @@ -195,29 +193,29 @@ static int ucode_status ( struct ucode_update *update, } /* Check microcode was not downgraded */ - if ( status.after < status.before ) { + if ( status->after < status->before ) { DBGC ( update, "UCODE %#08x was downgraded %#08x->%#08x\n", - id, status.before, status.after ); + id, status->before, status->after ); return -ENOTTY; } /* Check that expected updates (if any) were applied */ for ( desc = update->desc ; desc->signature ; desc++ ) { - if ( ( desc->signature == status.signature ) && - ( status.after < desc->version ) ) { + if ( ( desc->signature == status->signature ) && + ( status->after < desc->version ) ) { DBGC ( update, "UCODE %#08x failed update %#08x->%#08x " - "(wanted %#08x)\n", id, status.before, - status.after, desc->version ); + "(wanted %#08x)\n", id, status->before, + status->after, desc->version ); return -EIO; } } /* Update summary */ summary->count++; - if ( status.before < summary->low ) - summary->low = status.before; - if ( status.after > summary->high ) - summary->high = status.after; + if ( status->before < summary->low ) + summary->low = status->before; + if ( status->after > summary->high ) + summary->high = status->after; return 0; } @@ -231,13 +229,13 @@ static int ucode_status ( struct ucode_update *update, * @ret rc Return status code */ static int ucode_update_all ( struct image *image, - struct ucode_update *update, + const struct ucode_update *update, struct ucode_summary *summary ) { struct ucode_control control; struct ucode_vendor *vendor; - userptr_t status; + struct ucode_status *status; unsigned int max; - unsigned int i; + unsigned int id; size_t len; int rc; @@ -248,7 +246,7 @@ static int ucode_update_all ( struct image *image, /* Allocate status reports */ max = mp_max_cpuid(); - len = ( ( max + 1 ) * sizeof ( struct ucode_status ) ); + len = ( ( max + 1 ) * sizeof ( *status ) ); status = umalloc ( len ); if ( ! status ) { DBGC ( image, "UCODE %s could not allocate %d status reports\n", @@ -256,12 +254,12 @@ static int ucode_update_all ( struct image *image, rc = -ENOMEM; goto err_alloc; } - memset_user ( status, 0, 0, len ); + memset ( status, 0, len ); /* Construct control structure */ memset ( &control, 0, sizeof ( control ) ); control.desc = virt_to_phys ( update->desc ); - control.status = user_to_phys ( status, 0 ); + control.status = virt_to_phys ( status ); vendor = update->vendor; if ( vendor ) { control.ver_clear = vendor->ver_clear; @@ -274,8 +272,9 @@ static int ucode_update_all ( struct image *image, /* Update microcode on boot processor */ mp_exec_boot ( ucode_update, &control ); - if ( ( rc = ucode_status ( update, &control, summary, - mp_boot_cpuid(), 0 ) ) != 0 ) { + id = mp_boot_cpuid(); + if ( ( rc = ucode_status ( update, &control, &status[id], + summary, id, 0 ) ) != 0 ) { DBGC ( image, "UCODE %s failed on boot processor: %s\n", image->name, strerror ( rc ) ); goto err_boot; @@ -293,9 +292,9 @@ static int ucode_update_all ( struct image *image, /* Check status reports */ summary->count = 0; - for ( i = 0 ; i <= max ; i++ ) { - if ( ( rc = ucode_status ( update, &control, summary, - i, 1 ) ) != 0 ) { + for ( id = 0 ; id <= max ; id++ ) { + if ( ( rc = ucode_status ( update, &control, &status[id], + summary, id, 1 ) ) != 0 ) { goto err_status; } } @@ -359,24 +358,22 @@ static void ucode_describe ( struct image *image, size_t start, * @ret rc Return status code */ static int ucode_verify ( struct image *image, size_t start, size_t len ) { - uint32_t checksum = 0; - uint32_t dword; - size_t offset; + const uint32_t *dword; + uint32_t checksum; + unsigned int count; /* Check length is a multiple of dwords */ - if ( ( len % sizeof ( dword ) ) != 0 ) { + if ( ( len % sizeof ( *dword ) ) != 0 ) { DBGC ( image, "UCODE %s+%#04zx invalid length %#zx\n", image->name, start, len ); return -EINVAL; } + dword = ( image->data + start ); /* Calculate checksum */ - for ( offset = start ; len ; - offset += sizeof ( dword ), len -= sizeof ( dword ) ) { - copy_from_user ( &dword, image->data, offset, - sizeof ( dword ) ); - checksum += dword; - } + count = ( len / sizeof ( *dword ) ); + for ( checksum = 0 ; count ; count-- ) + checksum += *(dword++); if ( checksum != 0 ) { DBGC ( image, "UCODE %s+%#04zx bad checksum %#08x\n", image->name, start, checksum ); @@ -396,9 +393,9 @@ static int ucode_verify ( struct image *image, size_t start, size_t len ) { */ static int ucode_parse_intel ( struct image *image, size_t start, struct ucode_update *update ) { - struct intel_ucode_header hdr; - struct intel_ucode_ext_header exthdr; - struct intel_ucode_ext ext; + const struct intel_ucode_header *hdr; + const struct intel_ucode_ext_header *exthdr; + const struct intel_ucode_ext *ext; struct ucode_descriptor desc; size_t remaining; size_t offset; @@ -409,27 +406,27 @@ static int ucode_parse_intel ( struct image *image, size_t start, /* Read header */ remaining = ( image->len - start ); - if ( remaining < sizeof ( hdr ) ) { + if ( remaining < sizeof ( *hdr ) ) { DBGC ( image, "UCODE %s+%#04zx too small for Intel header\n", image->name, start ); return -ENOEXEC; } - copy_from_user ( &hdr, image->data, start, sizeof ( hdr ) ); + hdr = ( image->data + start ); /* Determine lengths */ - data_len = hdr.data_len; + data_len = hdr->data_len; if ( ! data_len ) data_len = INTEL_UCODE_DATA_LEN; - len = hdr.len; + len = hdr->len; if ( ! len ) - len = ( sizeof ( hdr ) + data_len ); + len = ( sizeof ( *hdr ) + data_len ); /* Verify a selection of fields */ - if ( ( hdr.hver != INTEL_UCODE_HVER ) || - ( hdr.lver != INTEL_UCODE_LVER ) || - ( len < sizeof ( hdr ) ) || + if ( ( hdr->hver != INTEL_UCODE_HVER ) || + ( hdr->lver != INTEL_UCODE_LVER ) || + ( len < sizeof ( *hdr ) ) || ( len > remaining ) || - ( data_len > ( len - sizeof ( hdr ) ) ) || + ( data_len > ( len - sizeof ( *hdr ) ) ) || ( ( data_len % sizeof ( uint32_t ) ) != 0 ) || ( ( len % INTEL_UCODE_ALIGN ) != 0 ) ) { DBGC2 ( image, "UCODE %s+%#04zx is not an Intel update\n", @@ -444,48 +441,46 @@ static int ucode_parse_intel ( struct image *image, size_t start, return rc; /* Populate descriptor */ - desc.signature = hdr.signature; - desc.version = hdr.version; - desc.address = user_to_phys ( image->data, - ( start + sizeof ( hdr ) ) ); + desc.signature = hdr->signature; + desc.version = hdr->version; + desc.address = ( virt_to_phys ( image->data ) + start + + sizeof ( *hdr ) ); /* Add non-extended descriptor, if applicable */ - ucode_describe ( image, start, &ucode_intel, &desc, hdr.platforms, + ucode_describe ( image, start, &ucode_intel, &desc, hdr->platforms, update ); /* Construct extended descriptors, if applicable */ - offset = ( sizeof ( hdr ) + data_len ); - if ( offset <= ( len - sizeof ( exthdr ) ) ) { + offset = ( sizeof ( *hdr ) + data_len ); + if ( offset <= ( len - sizeof ( *exthdr ) ) ) { /* Read extended header */ - copy_from_user ( &exthdr, image->data, ( start + offset ), - sizeof ( exthdr ) ); - offset += sizeof ( exthdr ); + exthdr = ( image->data + start + offset ); + offset += sizeof ( *exthdr ); /* Read extended signatures */ - for ( i = 0 ; i < exthdr.count ; i++ ) { + for ( i = 0 ; i < exthdr->count ; i++ ) { /* Read extended signature */ - if ( offset > ( len - sizeof ( ext ) ) ) { + if ( offset > ( len - sizeof ( *ext ) ) ) { DBGC ( image, "UCODE %s+%#04zx extended " "signature overrun\n", image->name, start ); return -EINVAL; } - copy_from_user ( &ext, image->data, ( start + offset ), - sizeof ( ext ) ); - offset += sizeof ( ext ); + ext = ( image->data + start + offset ); + offset += sizeof ( *ext ); /* Avoid duplicating non-extended descriptor */ - if ( ( ext.signature == hdr.signature ) && - ( ext.platforms == hdr.platforms ) ) { + if ( ( ext->signature == hdr->signature ) && + ( ext->platforms == hdr->platforms ) ) { continue; } /* Construct descriptor, if applicable */ - desc.signature = ext.signature; + desc.signature = ext->signature; ucode_describe ( image, start, &ucode_intel, &desc, - ext.platforms, update ); + ext->platforms, update ); } } @@ -502,10 +497,10 @@ static int ucode_parse_intel ( struct image *image, size_t start, */ static int ucode_parse_amd ( struct image *image, size_t start, struct ucode_update *update ) { - struct amd_ucode_header hdr; - struct amd_ucode_equivalence equiv; - struct amd_ucode_patch_header phdr; - struct amd_ucode_patch patch; + const struct amd_ucode_header *hdr; + const struct amd_ucode_equivalence *equiv; + const struct amd_ucode_patch_header *phdr; + const struct amd_ucode_patch *patch; struct ucode_descriptor desc; size_t remaining; size_t offset; @@ -515,91 +510,85 @@ static int ucode_parse_amd ( struct image *image, size_t start, /* Read header */ remaining = ( image->len - start ); - if ( remaining < sizeof ( hdr ) ) { + if ( remaining < sizeof ( *hdr ) ) { DBGC ( image, "UCODE %s+%#04zx too small for AMD header\n", image->name, start ); return -ENOEXEC; } - copy_from_user ( &hdr, image->data, start, sizeof ( hdr ) ); + hdr = ( image->data + start ); /* Check header */ - if ( hdr.magic != AMD_UCODE_MAGIC ) { + if ( hdr->magic != AMD_UCODE_MAGIC ) { DBGC2 ( image, "UCODE %s+%#04zx is not an AMD update\n", image->name, start ); return -ENOEXEC; } DBGC2 ( image, "UCODE %s+%#04zx is an AMD update\n", image->name, start ); - if ( hdr.type != AMD_UCODE_EQUIV_TYPE ) { + if ( hdr->type != AMD_UCODE_EQUIV_TYPE ) { DBGC ( image, "UCODE %s+%#04zx unsupported equivalence table " - "type %d\n", image->name, start, hdr.type ); + "type %d\n", image->name, start, hdr->type ); return -ENOTSUP; } - if ( hdr.len > ( remaining - sizeof ( hdr ) ) ) { + if ( hdr->len > ( remaining - sizeof ( *hdr ) ) ) { DBGC ( image, "UCODE %s+%#04zx truncated equivalence table\n", image->name, start ); return -EINVAL; } /* Count number of equivalence table entries */ - offset = sizeof ( hdr ); - for ( count = 0 ; offset < ( sizeof ( hdr ) + hdr.len ) ; - count++, offset += sizeof ( equiv ) ) { - copy_from_user ( &equiv, image->data, ( start + offset ), - sizeof ( equiv ) ); - if ( ! equiv.signature ) + offset = sizeof ( *hdr ); + equiv = ( image->data + start + offset ); + for ( count = 0 ; offset < ( sizeof ( *hdr ) + hdr->len ) ; + count++, offset += sizeof ( *equiv ) ) { + if ( ! equiv[count].signature ) break; } DBGC2 ( image, "UCODE %s+%#04zx has %d equivalence table entries\n", image->name, start, count ); /* Parse available updates */ - offset = ( sizeof ( hdr ) + hdr.len ); + offset = ( sizeof ( *hdr ) + hdr->len ); used = 0; while ( used < count ) { /* Read patch header */ - if ( ( offset + sizeof ( phdr ) ) > remaining ) { + if ( ( offset + sizeof ( *phdr ) ) > remaining ) { DBGC ( image, "UCODE %s+%#04zx truncated patch " "header\n", image->name, start ); return -EINVAL; } - copy_from_user ( &phdr, image->data, ( start + offset ), - sizeof ( phdr ) ); - offset += sizeof ( phdr ); + phdr = ( image->data + start + offset ); + offset += sizeof ( *phdr ); /* Validate patch header */ - if ( phdr.type != AMD_UCODE_PATCH_TYPE ) { + if ( phdr->type != AMD_UCODE_PATCH_TYPE ) { DBGC ( image, "UCODE %s+%#04zx unsupported patch type " - "%d\n", image->name, start, phdr.type ); + "%d\n", image->name, start, phdr->type ); return -ENOTSUP; } - if ( phdr.len < sizeof ( patch ) ) { + if ( phdr->len < sizeof ( *patch ) ) { DBGC ( image, "UCODE %s+%#04zx underlength patch\n", image->name, start ); return -EINVAL; } - if ( phdr.len > ( remaining - offset ) ) { + if ( phdr->len > ( remaining - offset ) ) { DBGC ( image, "UCODE %s+%#04zx truncated patch\n", image->name, start ); return -EINVAL; } /* Read patch and construct descriptor */ - copy_from_user ( &patch, image->data, ( start + offset ), - sizeof ( patch ) ); - desc.version = patch.version; - desc.address = user_to_phys ( image->data, ( start + offset ) ); - offset += phdr.len; + patch = ( image->data + start + offset ); + desc.version = patch->version; + desc.address = ( virt_to_phys ( image->data ) + + start + offset ); + offset += phdr->len; /* Parse equivalence table to find matching signatures */ for ( i = 0 ; i < count ; i++ ) { - copy_from_user ( &equiv, image->data, - ( start + sizeof ( hdr ) + - ( i * ( sizeof ( equiv ) ) ) ), - sizeof ( equiv ) ); - if ( patch.id == equiv.id ) { - desc.signature = equiv.signature; + if ( patch->id == equiv[i].id ) { + desc.signature = equiv[i].signature; ucode_describe ( image, start, &ucode_amd, &desc, 0, update ); used++; @@ -744,19 +733,19 @@ static int ucode_exec ( struct image *image ) { * @ret rc Return status code */ static int ucode_probe ( struct image *image ) { - union { + const union { struct intel_ucode_header intel; struct amd_ucode_header amd; - } header; + } *header; /* Sanity check */ - if ( image->len < sizeof ( header ) ) { + if ( image->len < sizeof ( *header ) ) { DBGC ( image, "UCODE %s too short\n", image->name ); return -ENOEXEC; } /* Read first microcode image header */ - copy_from_user ( &header, image->data, 0, sizeof ( header ) ); + header = image->data; /* Check for something that looks like an Intel update * @@ -769,19 +758,19 @@ static int ucode_probe ( struct image *image ) { * the image, and do not want to have a microcode image * erroneously treated as a PXE boot executable. */ - if ( ( header.intel.hver == INTEL_UCODE_HVER ) && - ( header.intel.lver == INTEL_UCODE_LVER ) && - ( ( header.intel.date.century == 0x19 ) || - ( ( header.intel.date.century >= 0x20 ) && - ( header.intel.date.century <= 0x29 ) ) ) ) { + if ( ( header->intel.hver == INTEL_UCODE_HVER ) && + ( header->intel.lver == INTEL_UCODE_LVER ) && + ( ( header->intel.date.century == 0x19 ) || + ( ( header->intel.date.century >= 0x20 ) && + ( header->intel.date.century <= 0x29 ) ) ) ) { DBGC ( image, "UCODE %s+%#04zx looks like an Intel update\n", image->name, ( ( size_t ) 0 ) ); return 0; } /* Check for AMD update signature */ - if ( ( header.amd.magic == AMD_UCODE_MAGIC ) && - ( header.amd.type == AMD_UCODE_EQUIV_TYPE ) ) { + if ( ( header->amd.magic == AMD_UCODE_MAGIC ) && + ( header->amd.type == AMD_UCODE_EQUIV_TYPE ) ) { DBGC ( image, "UCODE %s+%#04zx looks like an AMD update\n", image->name, ( ( size_t ) 0 ) ); return 0; diff --git a/src/arch/x86/include/basemem.h b/src/arch/x86/include/basemem.h index 01c2ea917..9ac918ac0 100644 --- a/src/arch/x86/include/basemem.h +++ b/src/arch/x86/include/basemem.h @@ -27,9 +27,4 @@ static inline unsigned int get_fbms ( void ) { extern void set_fbms ( unsigned int new_fbms ); -/* Actually in hidemem.c, but putting it here avoids polluting the - * architecture-independent include/hidemem.h. - */ -extern void hide_basemem ( void ); - #endif /* _BASEMEM_H */ diff --git a/src/arch/x86/include/bios_disks.h b/src/arch/x86/include/bios_disks.h deleted file mode 100644 index 0dd7c4ebb..000000000 --- a/src/arch/x86/include/bios_disks.h +++ /dev/null @@ -1,69 +0,0 @@ -#ifndef BIOS_DISKS_H -#define BIOS_DISKS_H - -#include "dev.h" - -/* - * Constants - * - */ - -#define BIOS_DISK_MAX_NAME_LEN 6 - -struct bios_disk_sector { - char data[512]; -}; - -/* - * The location of a BIOS disk - * - */ -struct bios_disk_loc { - uint8_t drive; -}; - -/* - * A physical BIOS disk device - * - */ -struct bios_disk_device { - char name[BIOS_DISK_MAX_NAME_LEN]; - uint8_t drive; - uint8_t type; -}; - -/* - * A BIOS disk driver, with a valid device ID range and naming - * function. - * - */ -struct bios_disk_driver { - void ( *fill_drive_name ) ( char *buf, uint8_t drive ); - uint8_t min_drive; - uint8_t max_drive; -}; - -/* - * Define a BIOS disk driver - * - */ -#define BIOS_DISK_DRIVER( _name, _fill_drive_name, _min_drive, _max_drive ) \ - static struct bios_disk_driver _name = { \ - .fill_drive_name = _fill_drive_name, \ - .min_drive = _min_drive, \ - .max_drive = _max_drive, \ - } - -/* - * Functions in bios_disks.c - * - */ - - -/* - * bios_disk bus global definition - * - */ -extern struct bus_driver bios_disk_driver; - -#endif /* BIOS_DISKS_H */ diff --git a/src/arch/x86/include/bits/acpi.h b/src/arch/x86/include/bits/acpi.h index a6ff90804..287bdafeb 100644 --- a/src/arch/x86/include/bits/acpi.h +++ b/src/arch/x86/include/bits/acpi.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/rsdp.h> diff --git a/src/arch/x86/include/bits/bigint.h b/src/arch/x86/include/bits/bigint.h index a6bc2ca1d..21cffa0cf 100644 --- a/src/arch/x86/include/bits/bigint.h +++ b/src/arch/x86/include/bits/bigint.h @@ -7,6 +7,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <stdint.h> #include <string.h> @@ -32,10 +33,12 @@ bigint_init_raw ( uint32_t *value0, unsigned int size, long discard_c; /* Copy raw data in reverse order, padding with zeros */ - __asm__ __volatile__ ( "\n1:\n\t" + __asm__ __volatile__ ( "jecxz 2f\n\t" + "\n1:\n\t" "movb -1(%3,%1), %%al\n\t" "stosb\n\t" "loop 1b\n\t" + "\n2:\n\t" "xorl %%eax, %%eax\n\t" "mov %4, %1\n\t" "rep stosb\n\t" @@ -52,8 +55,9 @@ bigint_init_raw ( uint32_t *value0, unsigned int size, * @v addend0 Element 0 of big integer to add * @v value0 Element 0 of big integer to be added to * @v size Number of elements + * @ret carry Carry flag */ -static inline __attribute__ (( always_inline )) void +static inline __attribute__ (( always_inline )) int bigint_add_raw ( const uint32_t *addend0, uint32_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = @@ -61,17 +65,20 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0, long index; void *discard_S; long discard_c; + int carry; __asm__ __volatile__ ( "xor %0, %0\n\t" /* Zero %0 and clear CF */ "\n1:\n\t" "lodsl\n\t" - "adcl %%eax, (%4,%0,4)\n\t" + "adcl %%eax, (%5,%0,4)\n\t" "inc %0\n\t" /* Does not affect CF */ "loop 1b\n\t" : "=&r" ( index ), "=&S" ( discard_S ), - "=&c" ( discard_c ), "+m" ( *value ) + "=&c" ( discard_c ), "=@ccc" ( carry ), + "+m" ( *value ) : "r" ( value0 ), "1" ( addend0 ), "2" ( size ) : "eax" ); + return carry; } /** @@ -80,8 +87,9 @@ bigint_add_raw ( const uint32_t *addend0, uint32_t *value0, * @v subtrahend0 Element 0 of big integer to subtract * @v value0 Element 0 of big integer to be subtracted from * @v size Number of elements + * @ret borrow Borrow flag */ -static inline __attribute__ (( always_inline )) void +static inline __attribute__ (( always_inline )) int bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = @@ -89,61 +97,71 @@ bigint_subtract_raw ( const uint32_t *subtrahend0, uint32_t *value0, long index; void *discard_S; long discard_c; + int borrow; __asm__ __volatile__ ( "xor %0, %0\n\t" /* Zero %0 and clear CF */ "\n1:\n\t" "lodsl\n\t" - "sbbl %%eax, (%4,%0,4)\n\t" + "sbbl %%eax, (%5,%0,4)\n\t" "inc %0\n\t" /* Does not affect CF */ "loop 1b\n\t" : "=&r" ( index ), "=&S" ( discard_S ), - "=&c" ( discard_c ), "+m" ( *value ) + "=&c" ( discard_c ), "=@ccc" ( borrow ), + "+m" ( *value ) : "r" ( value0 ), "1" ( subtrahend0 ), "2" ( size ) : "eax" ); + return borrow; } /** - * Rotate big integer left + * Shift big integer left * * @v value0 Element 0 of big integer * @v size Number of elements + * @ret out Bit shifted out */ -static inline __attribute__ (( always_inline )) void -bigint_rol_raw ( uint32_t *value0, unsigned int size ) { +static inline __attribute__ (( always_inline )) int +bigint_shl_raw ( uint32_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = ( ( void * ) value0 ); long index; long discard_c; + int out; __asm__ __volatile__ ( "xor %0, %0\n\t" /* Zero %0 and clear CF */ "\n1:\n\t" - "rcll $1, (%3,%0,4)\n\t" + "rcll $1, (%4,%0,4)\n\t" "inc %0\n\t" /* Does not affect CF */ "loop 1b\n\t" : "=&r" ( index ), "=&c" ( discard_c ), - "+m" ( *value ) + "=@ccc" ( out ), "+m" ( *value ) : "r" ( value0 ), "1" ( size ) ); + return out; } /** - * Rotate big integer right + * Shift big integer right * * @v value0 Element 0 of big integer * @v size Number of elements + * @ret out Bit shifted out */ -static inline __attribute__ (( always_inline )) void -bigint_ror_raw ( uint32_t *value0, unsigned int size ) { +static inline __attribute__ (( always_inline )) int +bigint_shr_raw ( uint32_t *value0, unsigned int size ) { bigint_t ( size ) __attribute__ (( may_alias )) *value = ( ( void * ) value0 ); long discard_c; + int out; __asm__ __volatile__ ( "clc\n\t" "\n1:\n\t" - "rcrl $1, -4(%2,%0,4)\n\t" + "rcrl $1, -4(%3,%0,4)\n\t" "loop 1b\n\t" - : "=&c" ( discard_c ), "+m" ( *value ) + : "=&c" ( discard_c ), "=@ccc" ( out ), + "+m" ( *value ) : "r" ( value0 ), "0" ( size ) ); + return out; } /** @@ -196,25 +214,6 @@ bigint_is_geq_raw ( const uint32_t *value0, const uint32_t *reference0, } /** - * Test if bit is set in big integer - * - * @v value0 Element 0 of big integer - * @v size Number of elements - * @v bit Bit to test - * @ret is_set Bit is set - */ -static inline __attribute__ (( always_inline )) int -bigint_bit_is_set_raw ( const uint32_t *value0, unsigned int size, - unsigned int bit ) { - const bigint_t ( size ) __attribute__ (( may_alias )) *value = - ( ( const void * ) value0 ); - unsigned int index = ( bit / ( 8 * sizeof ( value->element[0] ) ) ); - unsigned int subindex = ( bit % ( 8 * sizeof ( value->element[0] ) ) ); - - return ( value->element[index] & ( 1 << subindex ) ); -} - -/** * Find highest bit set in big integer * * @v value0 Element 0 of big integer @@ -312,20 +311,45 @@ bigint_done_raw ( const uint32_t *value0, unsigned int size __unused, long discard_c; /* Copy raw data in reverse order */ - __asm__ __volatile__ ( "\n1:\n\t" + __asm__ __volatile__ ( "jecxz 2f\n\t" + "\n1:\n\t" "movb -1(%3,%1), %%al\n\t" "stosb\n\t" "loop 1b\n\t" + "\n2:\n\t" : "=&D" ( discard_D ), "=&c" ( discard_c ), "+m" ( *out_bytes ) : "r" ( value0 ), "0" ( out ), "1" ( len ) : "eax" ); } -extern void bigint_multiply_raw ( const uint32_t *multiplicand0, - unsigned int multiplicand_size, - const uint32_t *multiplier0, - unsigned int multiplier_size, - uint32_t *value0 ); +/** + * Multiply big integer elements + * + * @v multiplicand Multiplicand element + * @v multiplier Multiplier element + * @v result Result element + * @v carry Carry element + */ +static inline __attribute__ (( always_inline )) void +bigint_multiply_one ( const uint32_t multiplicand, const uint32_t multiplier, + uint32_t *result, uint32_t *carry ) { + uint32_t discard_a; + + __asm__ __volatile__ ( /* Perform multiplication */ + "mull %3\n\t" + /* Accumulate carry */ + "addl %5, %0\n\t" + "adcl $0, %1\n\t" + /* Accumulate result */ + "addl %0, %2\n\t" + "adcl $0, %1\n\t" + : "=&a" ( discard_a ), + "=&d" ( *carry ), + "+m" ( *result ) + : "g" ( multiplicand ), + "0" ( multiplier ), + "r" ( *carry ) ); +} #endif /* _BITS_BIGINT_H */ diff --git a/src/arch/x86/include/bits/bitops.h b/src/arch/x86/include/bits/bitops.h index f697b8c8f..cdbc3b0a2 100644 --- a/src/arch/x86/include/bits/bitops.h +++ b/src/arch/x86/include/bits/bitops.h @@ -14,6 +14,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <stdint.h> diff --git a/src/arch/x86/include/bits/endian.h b/src/arch/x86/include/bits/endian.h index 85718cfdd..72279117d 100644 --- a/src/arch/x86/include/bits/endian.h +++ b/src/arch/x86/include/bits/endian.h @@ -2,6 +2,7 @@ #define _BITS_ENDIAN_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #define __BYTE_ORDER __LITTLE_ENDIAN diff --git a/src/arch/x86/include/bits/errfile.h b/src/arch/x86/include/bits/errfile.h index 78b3dea1c..e7aec6f39 100644 --- a/src/arch/x86/include/bits/errfile.h +++ b/src/arch/x86/include/bits/errfile.h @@ -2,14 +2,14 @@ #define _BITS_ERRFILE_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /** * @addtogroup errfile Error file identifiers * @{ */ -#define ERRFILE_memtop_umalloc ( ERRFILE_ARCH | ERRFILE_CORE | 0x00000000 ) -#define ERRFILE_memmap ( ERRFILE_ARCH | ERRFILE_CORE | 0x00010000 ) +#define ERRFILE_int15 ( ERRFILE_ARCH | ERRFILE_CORE | 0x00010000 ) #define ERRFILE_pnpbios ( ERRFILE_ARCH | ERRFILE_CORE | 0x00020000 ) #define ERRFILE_bios_smbios ( ERRFILE_ARCH | ERRFILE_CORE | 0x00030000 ) #define ERRFILE_biosint ( ERRFILE_ARCH | ERRFILE_CORE | 0x00040000 ) @@ -42,7 +42,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #define ERRFILE_comboot_resolv ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x00090000 ) #define ERRFILE_comboot_call ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000a0000 ) #define ERRFILE_sdi ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000b0000 ) -#define ERRFILE_initrd ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000c0000 ) #define ERRFILE_pxe_call ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000d0000 ) #define ERRFILE_ucode ( ERRFILE_ARCH | ERRFILE_IMAGE | 0x000e0000 ) diff --git a/src/arch/x86/include/bits/io.h b/src/arch/x86/include/bits/io.h index 95673ad8d..cde0b6829 100644 --- a/src/arch/x86/include/bits/io.h +++ b/src/arch/x86/include/bits/io.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /** Page shift */ #define PAGE_SHIFT 12 diff --git a/src/arch/x86/include/bits/iomap.h b/src/arch/x86/include/bits/iomap.h index d6fff257e..d524bd805 100644 --- a/src/arch/x86/include/bits/iomap.h +++ b/src/arch/x86/include/bits/iomap.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/iomap_pages.h> diff --git a/src/arch/x86/include/bits/memmap.h b/src/arch/x86/include/bits/memmap.h new file mode 100644 index 000000000..e68550fb8 --- /dev/null +++ b/src/arch/x86/include/bits/memmap.h @@ -0,0 +1,15 @@ +#ifndef _BITS_MEMMAP_H +#define _BITS_MEMMAP_H + +/** @file + * + * x86-specific system memory map API implementations + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); + +#include <ipxe/int15.h> + +#endif /* _BITS_MEMMAP_H */ diff --git a/src/arch/x86/include/bits/nap.h b/src/arch/x86/include/bits/nap.h index 7103b94c0..52c8d81ba 100644 --- a/src/arch/x86/include/bits/nap.h +++ b/src/arch/x86/include/bits/nap.h @@ -8,8 +8,16 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/bios_nap.h> -#include <ipxe/efi/efix86_nap.h> -#endif /* _BITS_MAP_H */ +/** + * Sleep until next CPU interrupt + * + */ +static inline __attribute__ (( always_inline )) void cpu_halt ( void ) { + __asm__ __volatile__ ( "hlt" ); +} + +#endif /* _BITS_NAP_H */ diff --git a/src/arch/x86/include/bits/ns16550.h b/src/arch/x86/include/bits/ns16550.h new file mode 100644 index 000000000..dbb1cd51c --- /dev/null +++ b/src/arch/x86/include/bits/ns16550.h @@ -0,0 +1,60 @@ +#ifndef _BITS_NS16550_H +#define _BITS_NS16550_H + +/** @file + * + * 16550-compatible UART + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); + +#include <stdint.h> +#include <ipxe/io.h> + +/** + * Write to UART register + * + * @v ns16550 16550 UART + * @v address Register address + * @v data Data + */ +static inline __attribute__ (( always_inline )) void +ns16550_write ( struct ns16550_uart *ns16550, unsigned int address, + uint8_t data ) { + + iowrite8 ( data, ( ns16550->base + address ) ); +} + +/** + * Read from UART register + * + * @v ns16550 16550 UART + * @v address Register address + * @ret data Data + */ +static inline __attribute__ (( always_inline )) uint8_t +ns16550_read ( struct ns16550_uart *ns16550, unsigned int address ) { + + return ioread8 ( ns16550->base + address ); +} + +/* Fixed ISA serial port base addresses */ +#define COM1_BASE 0x3f8 +#define COM2_BASE 0x2f8 +#define COM3_BASE 0x3e8 +#define COM4_BASE 0x2e8 + +/* Fixed ISA serial ports */ +extern struct uart com1; +extern struct uart com2; +extern struct uart com3; +extern struct uart com4; + +/* Fixed ISA serial port names */ +#define COM1 &com1 +#define COM2 &com2 +#define COM3 &com3 +#define COM4 &com4 + +#endif /* _BITS_NS16550_H */ diff --git a/src/arch/x86/include/bits/pci_io.h b/src/arch/x86/include/bits/pci_io.h index a074d3370..b6c01e5c4 100644 --- a/src/arch/x86/include/bits/pci_io.h +++ b/src/arch/x86/include/bits/pci_io.h @@ -8,9 +8,9 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/pcibios.h> #include <ipxe/pcidirect.h> -#include <ipxe/pcicloud.h> #endif /* _BITS_PCI_IO_H */ diff --git a/src/arch/x86/include/bits/reboot.h b/src/arch/x86/include/bits/reboot.h index e702dd3d0..8d8d0b40e 100644 --- a/src/arch/x86/include/bits/reboot.h +++ b/src/arch/x86/include/bits/reboot.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/bios_reboot.h> diff --git a/src/arch/x86/include/bits/sanboot.h b/src/arch/x86/include/bits/sanboot.h index 1b9924e64..ff7b88d14 100644 --- a/src/arch/x86/include/bits/sanboot.h +++ b/src/arch/x86/include/bits/sanboot.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/bios_sanboot.h> diff --git a/src/arch/x86/include/bits/smbios.h b/src/arch/x86/include/bits/smbios.h index 9977c87ac..2be98d887 100644 --- a/src/arch/x86/include/bits/smbios.h +++ b/src/arch/x86/include/bits/smbios.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/bios_smbios.h> diff --git a/src/arch/x86/include/bits/string.h b/src/arch/x86/include/bits/string.h index c26fe30d5..8b2b3070b 100644 --- a/src/arch/x86/include/bits/string.h +++ b/src/arch/x86/include/bits/string.h @@ -25,6 +25,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /** @file * diff --git a/src/arch/x86/include/bits/tcpip.h b/src/arch/x86/include/bits/tcpip.h index 0ac55b1a0..52d032427 100644 --- a/src/arch/x86/include/bits/tcpip.h +++ b/src/arch/x86/include/bits/tcpip.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); extern uint16_t tcpip_continue_chksum ( uint16_t partial, const void *data, size_t len ); diff --git a/src/arch/x86/include/bits/time.h b/src/arch/x86/include/bits/time.h index 556d96f64..a4aa8cc6e 100644 --- a/src/arch/x86/include/bits/time.h +++ b/src/arch/x86/include/bits/time.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/rtc_time.h> diff --git a/src/arch/x86/include/bits/uaccess.h b/src/arch/x86/include/bits/uaccess.h deleted file mode 100644 index e9e7e5af5..000000000 --- a/src/arch/x86/include/bits/uaccess.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _BITS_UACCESS_H -#define _BITS_UACCESS_H - -/** @file - * - * x86-specific user access API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <librm.h> - -#endif /* _BITS_UACCESS_H */ diff --git a/src/arch/x86/include/bits/uart.h b/src/arch/x86/include/bits/uart.h deleted file mode 100644 index e09cd3f4c..000000000 --- a/src/arch/x86/include/bits/uart.h +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef _BITS_UART_H -#define _BITS_UART_H - -/** @file - * - * 16550-compatible UART - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <stdint.h> -#include <ipxe/io.h> - -/** - * Write to UART register - * - * @v uart UART - * @v addr Register address - * @v data Data - */ -static inline __attribute__ (( always_inline )) void -uart_write ( struct uart *uart, unsigned int addr, uint8_t data ) { - outb ( data, ( uart->base + addr ) ); -} - -/** - * Read from UART register - * - * @v uart UART - * @v addr Register address - * @ret data Data - */ -static inline __attribute__ (( always_inline )) uint8_t -uart_read ( struct uart *uart, unsigned int addr ) { - return inb ( uart->base + addr ); -} - -extern int uart_select ( struct uart *uart, unsigned int port ); - -#endif /* _BITS_UART_H */ diff --git a/src/arch/x86/include/bits/umalloc.h b/src/arch/x86/include/bits/umalloc.h deleted file mode 100644 index 5d1f554d8..000000000 --- a/src/arch/x86/include/bits/umalloc.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _BITS_UMALLOC_H -#define _BITS_UMALLOC_H - -/** @file - * - * x86-specific user memory allocation API implementations - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <ipxe/memtop_umalloc.h> - -#endif /* _BITS_UMALLOC_H */ diff --git a/src/arch/x86/include/bits/xen.h b/src/arch/x86/include/bits/xen.h index 3433cea1f..313bec254 100644 --- a/src/arch/x86/include/bits/xen.h +++ b/src/arch/x86/include/bits/xen.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /* Hypercall registers */ #ifdef __x86_64__ diff --git a/src/arch/x86/include/initrd.h b/src/arch/x86/include/initrd.h deleted file mode 100644 index 2fb9d3d3a..000000000 --- a/src/arch/x86/include/initrd.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef _INITRD_H -#define _INITRD_H - -/** @file - * - * Initial ramdisk (initrd) reshuffling - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <ipxe/uaccess.h> - -/** Minimum free space required to reshuffle initrds - * - * Chosen to avoid absurdly long reshuffling times - */ -#define INITRD_MIN_FREE_LEN ( 512 * 1024 ) - -extern void initrd_reshuffle ( userptr_t bottom ); -extern int initrd_reshuffle_check ( size_t len, userptr_t bottom ); - -#endif /* _INITRD_H */ diff --git a/src/arch/x86/include/ipxe/bios_nap.h b/src/arch/x86/include/ipxe/bios_nap.h index c9b82c1e5..7d94b3c4a 100644 --- a/src/arch/x86/include/ipxe/bios_nap.h +++ b/src/arch/x86/include/ipxe/bios_nap.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef NAP_PCBIOS #define NAP_PREFIX_pcbios diff --git a/src/arch/x86/include/ipxe/bios_reboot.h b/src/arch/x86/include/ipxe/bios_reboot.h index 3f6df9073..bd1bb42cc 100644 --- a/src/arch/x86/include/ipxe/bios_reboot.h +++ b/src/arch/x86/include/ipxe/bios_reboot.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef REBOOT_PCBIOS #define REBOOT_PREFIX_pcbios diff --git a/src/arch/x86/include/ipxe/bios_sanboot.h b/src/arch/x86/include/ipxe/bios_sanboot.h index 85d698039..d28339e4e 100644 --- a/src/arch/x86/include/ipxe/bios_sanboot.h +++ b/src/arch/x86/include/ipxe/bios_sanboot.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef SANBOOT_PCBIOS #define SANBOOT_PREFIX_pcbios diff --git a/src/arch/x86/include/ipxe/bios_smbios.h b/src/arch/x86/include/ipxe/bios_smbios.h index 9f7f9c8ff..1815e3617 100644 --- a/src/arch/x86/include/ipxe/bios_smbios.h +++ b/src/arch/x86/include/ipxe/bios_smbios.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef SMBIOS_PCBIOS #define SMBIOS_PREFIX_pcbios diff --git a/src/arch/x86/include/ipxe/cpuid.h b/src/arch/x86/include/ipxe/cpuid.h index 90d1bf01d..1851a859b 100644 --- a/src/arch/x86/include/ipxe/cpuid.h +++ b/src/arch/x86/include/ipxe/cpuid.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <stdint.h> @@ -33,6 +34,9 @@ struct x86_features { /** CPUID extended function */ #define CPUID_EXTENDED 0x80000000UL +/** CPUID hypervisor function */ +#define CPUID_HYPERVISOR 0x40000000UL + /** Get vendor ID and largest standard function */ #define CPUID_VENDOR_ID 0x00000000UL diff --git a/src/arch/x86/include/ipxe/efi/efix86_nap.h b/src/arch/x86/include/ipxe/efi/efix86_nap.h deleted file mode 100644 index 1a391c9b6..000000000 --- a/src/arch/x86/include/ipxe/efi/efix86_nap.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _IPXE_EFIX86_NAP_H -#define _IPXE_EFIX86_NAP_H - -/** @file - * - * EFI CPU sleeping - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#ifdef NAP_EFIX86 -#define NAP_PREFIX_efix86 -#else -#define NAP_PREFIX_efix86 __efix86_ -#endif - -#endif /* _IPXE_EFIX86_NAP_H */ diff --git a/src/arch/x86/include/ipxe/int15.h b/src/arch/x86/include/ipxe/int15.h new file mode 100644 index 000000000..590c0e9a7 --- /dev/null +++ b/src/arch/x86/include/ipxe/int15.h @@ -0,0 +1,22 @@ +#ifndef _IPXE_INT15_H +#define _IPXE_INT15_H + +/** @file + * + * INT15-based memory map + * + */ + +FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); + +#ifdef MEMMAP_INT15 +#define MEMMAP_PREFIX_int15 +#else +#define MEMMAP_PREFIX_int15 __int15_ +#endif + +extern void int15_intercept ( int intercept ); +extern void hide_basemem ( void ); + +#endif /* _IPXE_INT15_H */ diff --git a/src/arch/x86/include/ipxe/iomap_pages.h b/src/arch/x86/include/ipxe/iomap_pages.h index 18e0a3002..e74dabd90 100644 --- a/src/arch/x86/include/ipxe/iomap_pages.h +++ b/src/arch/x86/include/ipxe/iomap_pages.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef IOMAP_PAGES #define IOMAP_PREFIX_pages diff --git a/src/arch/x86/include/ipxe/memtop_umalloc.h b/src/arch/x86/include/ipxe/memtop_umalloc.h deleted file mode 100644 index dee055d16..000000000 --- a/src/arch/x86/include/ipxe/memtop_umalloc.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _IPXE_MEMTOP_UMALLOC_H -#define _IPXE_MEMTOP_UMALLOC_H - -/** @file - * - * External memory allocation - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#ifdef UMALLOC_MEMTOP -#define UMALLOC_PREFIX_memtop -#else -#define UMALLOC_PREFIX_memtop __memtop_ -#endif - -#endif /* _IPXE_MEMTOP_UMALLOC_H */ diff --git a/src/arch/x86/include/ipxe/pcibios.h b/src/arch/x86/include/ipxe/pcibios.h index 3caea1cfe..2fd03198e 100644 --- a/src/arch/x86/include/ipxe/pcibios.h +++ b/src/arch/x86/include/ipxe/pcibios.h @@ -10,6 +10,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef PCIAPI_PCBIOS #define PCIAPI_PREFIX_pcbios @@ -33,6 +34,17 @@ extern int pcibios_write ( struct pci_device *pci, uint32_t command, uint32_t value ); /** + * Check if PCI bus probing is allowed + * + * @v pci PCI device + * @ret ok Bus probing is allowed + */ +static inline __always_inline int +PCIAPI_INLINE ( pcbios, pci_can_probe ) ( struct pci_device *pci __unused ) { + return 1; +} + +/** * Read byte from PCI configuration space via PCI BIOS * * @v pci PCI device @@ -145,6 +157,4 @@ PCIAPI_INLINE ( pcbios, pci_ioremap ) ( struct pci_device *pci __unused, return ioremap ( bus_addr, len ); } -extern struct pci_api pcibios_api; - #endif /* _IPXE_PCIBIOS_H */ diff --git a/src/arch/x86/include/ipxe/pcicloud.h b/src/arch/x86/include/ipxe/pcicloud.h deleted file mode 100644 index 52268908c..000000000 --- a/src/arch/x86/include/ipxe/pcicloud.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _IPXE_PCICLOUD_H -#define _IPXE_PCICLOUD_H - -/** @file - * - * Cloud VM PCI configuration space access - * - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#ifdef PCIAPI_CLOUD -#define PCIAPI_PREFIX_cloud -#else -#define PCIAPI_PREFIX_cloud __cloud_ -#endif - -#endif /* _IPXE_PCICLOUD_H */ diff --git a/src/arch/x86/include/ipxe/pcidirect.h b/src/arch/x86/include/ipxe/pcidirect.h index 98c6a2bbb..5863b4d16 100644 --- a/src/arch/x86/include/ipxe/pcidirect.h +++ b/src/arch/x86/include/ipxe/pcidirect.h @@ -2,6 +2,7 @@ #define _PCIDIRECT_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <stdint.h> #include <ipxe/io.h> @@ -26,6 +27,17 @@ struct pci_device; extern void pcidirect_prepare ( struct pci_device *pci, int where ); /** + * Check if PCI bus probing is allowed + * + * @v pci PCI device + * @ret ok Bus probing is allowed + */ +static inline __always_inline int +PCIAPI_INLINE ( direct, pci_can_probe ) ( struct pci_device *pci __unused ) { + return 1; +} + +/** * Find next PCI bus:dev.fn address range in system * * @v busdevfn Starting PCI bus:dev.fn address @@ -155,6 +167,4 @@ PCIAPI_INLINE ( direct, pci_ioremap ) ( struct pci_device *pci __unused, return ioremap ( bus_addr, len ); } -extern struct pci_api pcidirect_api; - #endif /* _PCIDIRECT_H */ diff --git a/src/arch/x86/include/ipxe/rsdp.h b/src/arch/x86/include/ipxe/rsdp.h index 14afcd774..f371d9a20 100644 --- a/src/arch/x86/include/ipxe/rsdp.h +++ b/src/arch/x86/include/ipxe/rsdp.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef ACPI_RSDP #define ACPI_PREFIX_rsdp @@ -20,9 +21,9 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * * @v signature Requested table signature * @v index Requested index of table with this signature - * @ret table Table, or UNULL if not found + * @ret table Table, or NULL if not found */ -static inline __attribute__ (( always_inline )) userptr_t +static inline __attribute__ (( always_inline )) const struct acpi_header * ACPI_INLINE ( rsdp, acpi_find ) ( uint32_t signature, unsigned int index ) { return acpi_find_via_rsdt ( signature, index ); diff --git a/src/arch/x86/include/ipxe/rtc_time.h b/src/arch/x86/include/ipxe/rtc_time.h index cb8c7f49e..49c6313ed 100644 --- a/src/arch/x86/include/ipxe/rtc_time.h +++ b/src/arch/x86/include/ipxe/rtc_time.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef TIME_RTC #define TIME_PREFIX_rtc diff --git a/src/arch/x86/include/ipxe/x86_io.h b/src/arch/x86/include/ipxe/x86_io.h index eeb3f8454..164b57e92 100644 --- a/src/arch/x86/include/ipxe/x86_io.h +++ b/src/arch/x86/include/ipxe/x86_io.h @@ -16,6 +16,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #ifdef IOAPI_X86 #define IOAPI_PREFIX_x86 diff --git a/src/arch/x86/include/libkir.h b/src/arch/x86/include/libkir.h index 1f5b13504..76766b6c2 100644 --- a/src/arch/x86/include/libkir.h +++ b/src/arch/x86/include/libkir.h @@ -194,7 +194,7 @@ copy_from_user ( void *dest, userptr_t buffer, off_t offset, size_t len ) { * @ret buffer User buffer */ static inline __attribute__ (( always_inline )) userptr_t -real_to_user ( unsigned int segment, unsigned int offset ) { +real_to_virt ( unsigned int segment, unsigned int offset ) { return ( ( segment << 16 ) | offset ); } @@ -210,7 +210,7 @@ real_to_user ( unsigned int segment, unsigned int offset ) { */ static inline __attribute__ (( always_inline )) userptr_t virt_to_user ( void * virtual ) { - return real_to_user ( rm_ds, ( intptr_t ) virtual ); + return real_to_virt ( rm_ds, ( intptr_t ) virtual ); } /* TEXT16_CODE: declare a fragment of code that resides in .text16 */ diff --git a/src/arch/x86/include/librm.h b/src/arch/x86/include/librm.h index 84b345d3a..666be0438 100644 --- a/src/arch/x86/include/librm.h +++ b/src/arch/x86/include/librm.h @@ -64,12 +64,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #else /* ASSEMBLY */ -#ifdef UACCESS_LIBRM -#define UACCESS_PREFIX_librm -#else -#define UACCESS_PREFIX_librm __librm_ -#endif - /** * Call C function from real-mode code * @@ -79,114 +73,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); "pushl $( " _S2 ( VIRTUAL ( function ) ) " )\n\t" \ "call virt_call\n\t" -/* Variables in librm.S */ -extern const unsigned long virt_offset; - -/** - * Convert physical address to user pointer - * - * @v phys_addr Physical address - * @ret userptr User pointer - */ -static inline __always_inline userptr_t -UACCESS_INLINE ( librm, phys_to_user ) ( unsigned long phys_addr ) { - - /* In a 64-bit build, any valid physical address is directly - * usable as a virtual address, since the low 4GB is - * identity-mapped. - */ - if ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) - return phys_addr; - - /* In a 32-bit build, subtract virt_offset */ - return ( phys_addr - virt_offset ); -} - -/** - * Convert user buffer to physical address - * - * @v userptr User pointer - * @v offset Offset from user pointer - * @ret phys_addr Physical address - */ -static inline __always_inline unsigned long -UACCESS_INLINE ( librm, user_to_phys ) ( userptr_t userptr, off_t offset ) { - unsigned long addr = ( userptr + offset ); - - /* In a 64-bit build, any virtual address in the low 4GB is - * directly usable as a physical address, since the low 4GB is - * identity-mapped. - */ - if ( ( sizeof ( physaddr_t ) > sizeof ( uint32_t ) ) && - ( addr <= 0xffffffffUL ) ) - return addr; - - /* In a 32-bit build or in a 64-bit build with a virtual - * address above 4GB: add virt_offset - */ - return ( addr + virt_offset ); -} - -static inline __always_inline userptr_t -UACCESS_INLINE ( librm, virt_to_user ) ( volatile const void *addr ) { - return trivial_virt_to_user ( addr ); -} - -static inline __always_inline void * -UACCESS_INLINE ( librm, user_to_virt ) ( userptr_t userptr, off_t offset ) { - return trivial_user_to_virt ( userptr, offset ); -} - -static inline __always_inline userptr_t -UACCESS_INLINE ( librm, userptr_add ) ( userptr_t userptr, off_t offset ) { - return trivial_userptr_add ( userptr, offset ); -} - -static inline __always_inline off_t -UACCESS_INLINE ( librm, userptr_sub ) ( userptr_t userptr, - userptr_t subtrahend ) { - return trivial_userptr_sub ( userptr, subtrahend ); -} - -static inline __always_inline void -UACCESS_INLINE ( librm, memcpy_user ) ( userptr_t dest, off_t dest_off, - userptr_t src, off_t src_off, - size_t len ) { - trivial_memcpy_user ( dest, dest_off, src, src_off, len ); -} - -static inline __always_inline void -UACCESS_INLINE ( librm, memmove_user ) ( userptr_t dest, off_t dest_off, - userptr_t src, off_t src_off, - size_t len ) { - trivial_memmove_user ( dest, dest_off, src, src_off, len ); -} - -static inline __always_inline int -UACCESS_INLINE ( librm, memcmp_user ) ( userptr_t first, off_t first_off, - userptr_t second, off_t second_off, - size_t len ) { - return trivial_memcmp_user ( first, first_off, second, second_off, len); -} - -static inline __always_inline void -UACCESS_INLINE ( librm, memset_user ) ( userptr_t buffer, off_t offset, - int c, size_t len ) { - trivial_memset_user ( buffer, offset, c, len ); -} - -static inline __always_inline size_t -UACCESS_INLINE ( librm, strlen_user ) ( userptr_t buffer, off_t offset ) { - return trivial_strlen_user ( buffer, offset ); -} - -static inline __always_inline off_t -UACCESS_INLINE ( librm, memchr_user ) ( userptr_t buffer, off_t offset, - int c, size_t len ) { - return trivial_memchr_user ( buffer, offset, c, len ); -} - - /****************************************************************************** * * Access to variables in .data16 and .text16 @@ -244,8 +130,8 @@ extern const uint16_t __text16 ( rm_cs ); extern const uint16_t __text16 ( rm_ds ); #define rm_ds __use_text16 ( rm_ds ) -extern uint16_t copy_user_to_rm_stack ( userptr_t data, size_t size ); -extern void remove_user_from_rm_stack ( userptr_t data, size_t size ); +extern uint16_t copy_to_rm_stack ( const void *data, size_t size ); +extern void remove_from_rm_stack ( void *data, size_t size ); /* CODE_DEFAULT: restore default .code32/.code64 directive */ #ifdef __x86_64__ @@ -460,6 +346,10 @@ enum page_flags { /** The I/O space page table */ extern struct page_table io_pages; +/** Maximum number of I/O pages */ +#define IO_PAGE_COUNT \ + ( sizeof ( io_pages.page ) / sizeof ( io_pages.page[0] ) ) + /** I/O page size * * We choose to use 2MB pages for I/O space, to minimise the number of @@ -479,7 +369,8 @@ extern char __text16_array ( sipi, [] ); #define sipi __use_text16 ( sipi ) /** Length of startup IPI real-mode handler */ -extern char sipi_len[]; +extern size_t ABS_SYMBOL ( sipi_len ); +#define sipi_len ABS_VALUE ( sipi_len ) /** Startup IPI real-mode handler copy of real-mode data segment */ extern uint16_t __text16 ( sipi_ds ); diff --git a/src/arch/x86/include/pic8259.h b/src/arch/x86/include/pic8259.h index dbec5fd2c..0dc59cf27 100644 --- a/src/arch/x86/include/pic8259.h +++ b/src/arch/x86/include/pic8259.h @@ -47,9 +47,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); /* Macros to enable/disable IRQs */ #define IMR_REG(x) ( (x) < IRQ_PIC_CUTOFF ? PIC1_IMR : PIC2_IMR ) #define IMR_BIT(x) ( 1 << ( (x) % IRQ_PIC_CUTOFF ) ) -#define irq_enabled(x) ( ( inb ( IMR_REG(x) ) & IMR_BIT(x) ) == 0 ) -#define enable_irq(x) outb ( inb( IMR_REG(x) ) & ~IMR_BIT(x), IMR_REG(x) ) -#define disable_irq(x) outb ( inb( IMR_REG(x) ) | IMR_BIT(x), IMR_REG(x) ) /* Macros for acknowledging IRQs */ #define ICR_REG( irq ) ( (irq) < IRQ_PIC_CUTOFF ? PIC1_ICR : PIC2_ICR ) @@ -63,6 +60,50 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #define IRQ_MAX 15 #define IRQ_NONE -1U +/** + * Check if interrupt is enabled + * + * @v irq Interrupt number + * @ret enabled Interrupt is currently enabled + */ +static inline __attribute__ (( always_inline )) int +irq_enabled ( unsigned int irq ) { + int imr = inb ( IMR_REG ( irq ) ); + int mask = IMR_BIT ( irq ); + + return ( ( imr & mask ) == 0 ); +} + +/** + * Enable interrupt + * + * @v irq Interrupt number + * @ret enabled Interrupt was previously enabled + */ +static inline __attribute__ (( always_inline )) int +enable_irq ( unsigned int irq ) { + int imr = inb ( IMR_REG ( irq ) ); + int mask = IMR_BIT ( irq ); + + outb ( ( imr & ~mask ), IMR_REG ( irq ) ); + return ( ( imr & mask ) == 0 ); +} + +/** + * Disable interrupt + * + * @v irq Interrupt number + * @ret enabled Interrupt was previously enabled + */ +static inline __attribute__ (( always_inline )) int +disable_irq ( unsigned int irq ) { + int imr = inb ( IMR_REG ( irq ) ); + int mask = IMR_BIT ( irq ); + + outb ( ( imr | mask ), IMR_REG ( irq ) ); + return ( ( imr & mask ) == 0 ); +} + /* Function prototypes */ void send_eoi ( unsigned int irq ); diff --git a/src/arch/x86/include/pxe.h b/src/arch/x86/include/pxe.h index 54649b504..8e7aa1ce7 100644 --- a/src/arch/x86/include/pxe.h +++ b/src/arch/x86/include/pxe.h @@ -85,8 +85,6 @@ struct pxe_api_call { * @ret exit PXE API call exit code */ PXENV_EXIT_t ( * entry ) ( union u_PXENV_ANY *params ); - /** Length of parameters */ - uint16_t params_len; /** Opcode */ uint16_t opcode; }; @@ -112,7 +110,6 @@ struct pxe_api_call { ( union u_PXENV_ANY *params ) ) _entry ) \ : ( ( PXENV_EXIT_t ( * ) \ ( union u_PXENV_ANY *params ) ) _entry ) ), \ - .params_len = sizeof ( _params_type ), \ .opcode = _opcode, \ } diff --git a/src/arch/x86/include/realmode.h b/src/arch/x86/include/realmode.h index 4defd3b97..7baec56ca 100644 --- a/src/arch/x86/include/realmode.h +++ b/src/arch/x86/include/realmode.h @@ -2,7 +2,9 @@ #define REALMODE_H #include <stdint.h> +#include <string.h> #include <registers.h> +#include <librm.h> #include <ipxe/uaccess.h> /* @@ -65,15 +67,15 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); */ /** - * Convert segment:offset address to user buffer + * Convert segment:offset address to virtual address * * @v segment Real-mode segment * @v offset Real-mode offset - * @ret buffer User buffer + * @ret virt Virtual address */ -static inline __always_inline userptr_t -real_to_user ( unsigned int segment, unsigned int offset ) { - return ( phys_to_user ( ( segment << 4 ) + offset ) ); +static inline __always_inline void * +real_to_virt ( unsigned int segment, unsigned int offset ) { + return ( phys_to_virt ( ( segment << 4 ) + offset ) ); } /** @@ -87,7 +89,7 @@ real_to_user ( unsigned int segment, unsigned int offset ) { static inline __always_inline void copy_to_real ( unsigned int dest_seg, unsigned int dest_off, void *src, size_t n ) { - copy_to_user ( real_to_user ( dest_seg, dest_off ), 0, src, n ); + memcpy ( real_to_virt ( dest_seg, dest_off ), src, n ); } /** @@ -101,7 +103,7 @@ copy_to_real ( unsigned int dest_seg, unsigned int dest_off, static inline __always_inline void copy_from_real ( void *dest, unsigned int src_seg, unsigned int src_off, size_t n ) { - copy_from_user ( dest, real_to_user ( src_seg, src_off ), 0, n ); + memcpy ( dest, real_to_virt ( src_seg, src_off ), n ); } /** diff --git a/src/arch/x86/interface/pcbios/acpi_timer.c b/src/arch/x86/interface/pcbios/acpi_timer.c index 2e4047e38..e1523578b 100644 --- a/src/arch/x86/interface/pcbios/acpi_timer.c +++ b/src/arch/x86/interface/pcbios/acpi_timer.c @@ -102,20 +102,19 @@ static void acpi_udelay ( unsigned long usecs ) { * @ret rc Return status code */ static int acpi_timer_probe ( void ) { - struct acpi_fadt fadtab; - userptr_t fadt; + const struct acpi_fadt *fadt; unsigned int pm_tmr_blk; /* Locate FADT */ - fadt = acpi_table ( FADT_SIGNATURE, 0 ); + fadt = container_of ( acpi_table ( FADT_SIGNATURE, 0 ), + struct acpi_fadt, acpi ); if ( ! fadt ) { DBGC ( &acpi_timer, "ACPI could not find FADT\n" ); return -ENOENT; } /* Read FADT */ - copy_from_user ( &fadtab, fadt, 0, sizeof ( fadtab ) ); - pm_tmr_blk = le32_to_cpu ( fadtab.pm_tmr_blk ); + pm_tmr_blk = le32_to_cpu ( fadt->pm_tmr_blk ); if ( ! pm_tmr_blk ) { DBGC ( &acpi_timer, "ACPI has no timer\n" ); return -ENOENT; diff --git a/src/arch/x86/interface/pcbios/acpipwr.c b/src/arch/x86/interface/pcbios/acpipwr.c index f08b4af25..cb82ef1b4 100644 --- a/src/arch/x86/interface/pcbios/acpipwr.c +++ b/src/arch/x86/interface/pcbios/acpipwr.c @@ -23,6 +23,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +#include <string.h> #include <unistd.h> #include <errno.h> #include <byteswap.h> @@ -62,8 +63,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * uglier hacks I have ever implemented, but it's still prettier than * the ACPI specification itself. */ -static int acpi_extract_sx ( userptr_t zsdt, size_t len, size_t offset, - void *data ) { +static int acpi_extract_sx ( const struct acpi_header *zsdt, size_t len, + size_t offset, void *data ) { unsigned int *sx = data; uint8_t bytes[4]; uint8_t *byte; @@ -77,7 +78,8 @@ static int acpi_extract_sx ( userptr_t zsdt, size_t len, size_t offset, } /* Read first four bytes of value */ - copy_from_user ( bytes, zsdt, offset, sizeof ( bytes ) ); + memcpy ( bytes, ( ( ( const void * ) zsdt ) + offset ), + sizeof ( bytes ) ); DBGC ( colour, "ACPI found \\_Sx containing %02x:%02x:%02x:%02x\n", bytes[0], bytes[1], bytes[2], bytes[3] ); @@ -111,8 +113,7 @@ static int acpi_extract_sx ( userptr_t zsdt, size_t len, size_t offset, * @ret rc Return status code */ int acpi_poweroff ( void ) { - struct acpi_fadt fadtab; - userptr_t fadt; + const struct acpi_fadt *fadt; unsigned int pm1a_cnt_blk; unsigned int pm1b_cnt_blk; unsigned int pm1a_cnt; @@ -123,16 +124,16 @@ int acpi_poweroff ( void ) { int rc; /* Locate FADT */ - fadt = acpi_table ( FADT_SIGNATURE, 0 ); + fadt = container_of ( acpi_table ( FADT_SIGNATURE, 0 ), + struct acpi_fadt, acpi ); if ( ! fadt ) { DBGC ( colour, "ACPI could not find FADT\n" ); return -ENOENT; } /* Read FADT */ - copy_from_user ( &fadtab, fadt, 0, sizeof ( fadtab ) ); - pm1a_cnt_blk = le32_to_cpu ( fadtab.pm1a_cnt_blk ); - pm1b_cnt_blk = le32_to_cpu ( fadtab.pm1b_cnt_blk ); + pm1a_cnt_blk = le32_to_cpu ( fadt->pm1a_cnt_blk ); + pm1b_cnt_blk = le32_to_cpu ( fadt->pm1b_cnt_blk ); pm1a_cnt = ( pm1a_cnt_blk + ACPI_PM1_CNT ); pm1b_cnt = ( pm1b_cnt_blk + ACPI_PM1_CNT ); diff --git a/src/arch/x86/interface/pcbios/basemem.c b/src/arch/x86/interface/pcbios/basemem.c index 6a46081aa..8f3a30e92 100644 --- a/src/arch/x86/interface/pcbios/basemem.c +++ b/src/arch/x86/interface/pcbios/basemem.c @@ -27,7 +27,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <realmode.h> #include <bios.h> #include <basemem.h> -#include <ipxe/hidemem.h> +#include <ipxe/memmap.h> /** @file * diff --git a/src/arch/x86/interface/pcbios/bios_cachedhcp.c b/src/arch/x86/interface/pcbios/bios_cachedhcp.c index bea803d6e..60191c9c5 100644 --- a/src/arch/x86/interface/pcbios/bios_cachedhcp.c +++ b/src/arch/x86/interface/pcbios/bios_cachedhcp.c @@ -24,6 +24,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdint.h> +#include <string.h> #include <ipxe/init.h> #include <ipxe/cachedhcp.h> #include <realmode.h> @@ -60,7 +61,7 @@ static void cachedhcp_init ( void ) { /* Record cached DHCPACK */ if ( ( rc = cachedhcp_record ( &cached_dhcpack, 0, - phys_to_user ( cached_dhcpack_phys ), + phys_to_virt ( cached_dhcpack_phys ), sizeof ( BOOTPLAYER_t ) ) ) != 0 ) { DBGC ( colour, "CACHEDHCP could not record DHCPACK: %s\n", strerror ( rc ) ); @@ -73,5 +74,6 @@ static void cachedhcp_init ( void ) { /** Cached DHCPACK initialisation function */ struct init_fn cachedhcp_init_fn __init_fn ( INIT_NORMAL ) = { + .name = "cachedhcp", .initialise = cachedhcp_init, }; diff --git a/src/arch/x86/interface/pcbios/bios_mp.c b/src/arch/x86/interface/pcbios/bios_mp.c index 9e1179ccd..4525a60cf 100644 --- a/src/arch/x86/interface/pcbios/bios_mp.c +++ b/src/arch/x86/interface/pcbios/bios_mp.c @@ -30,6 +30,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); */ #include <registers.h> +#include <librm.h> #include <ipxe/uaccess.h> #include <ipxe/timer.h> #include <ipxe/msr.h> @@ -92,9 +93,9 @@ static void bios_mp_exec_boot ( mp_func_t func, void *opaque ) { "pushl %k1\n\t" "call *%k0\n\t" "addl $8, %%esp\n\t" ) - : : "r" ( mp_address ( mp_call ) ), - "r" ( mp_address ( func ) ), - "r" ( mp_address ( opaque ) ) ); + : : "R" ( mp_address ( mp_call ) ), + "R" ( mp_address ( func ) ), + "R" ( mp_address ( opaque ) ) ); } /** diff --git a/src/arch/x86/interface/pcbios/bios_reboot.c b/src/arch/x86/interface/pcbios/bios_reboot.c index 071173f19..c7f25405f 100644 --- a/src/arch/x86/interface/pcbios/bios_reboot.c +++ b/src/arch/x86/interface/pcbios/bios_reboot.c @@ -29,6 +29,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * */ +#include <string.h> #include <ipxe/reboot.h> #include <realmode.h> #include <bios.h> @@ -38,14 +39,14 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); /** * Reboot system * - * @v warm Perform a warm reboot + * @v flags Reboot flags */ -static void bios_reboot ( int warm ) { - uint16_t flag; +static void bios_reboot ( int flags ) { + uint16_t type; /* Configure BIOS for cold/warm reboot */ - flag = ( warm ? BDA_REBOOT_WARM : 0 ); - put_real ( flag, BDA_SEG, BDA_REBOOT ); + type = ( ( flags & REBOOT_WARM ) ? BDA_REBOOT_WARM : 0 ); + put_real ( type, BDA_SEG, BDA_REBOOT ); /* Jump to system reset vector */ __asm__ __volatile__ ( REAL_CODE ( "ljmp $0xf000, $0xfff0" ) : ); diff --git a/src/arch/x86/interface/pcbios/bios_smbios.c b/src/arch/x86/interface/pcbios/bios_smbios.c index 366679d36..aaec1eea1 100644 --- a/src/arch/x86/interface/pcbios/bios_smbios.c +++ b/src/arch/x86/interface/pcbios/bios_smbios.c @@ -45,19 +45,18 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * @ret rc Return status code */ static int bios_find_smbios2 ( struct smbios *smbios ) { - struct smbios_entry entry; - int rc; + const struct smbios_entry *entry; /* Scan through BIOS segment to find SMBIOS 32-bit entry point */ - if ( ( rc = find_smbios_entry ( real_to_user ( BIOS_SEG, 0 ), 0x10000, - &entry ) ) != 0 ) - return rc; + entry = find_smbios_entry ( real_to_virt ( BIOS_SEG, 0 ), 0x10000 ); + if ( ! entry ) + return -ENOENT; /* Fill in entry point descriptor structure */ - smbios->address = phys_to_user ( entry.smbios_address ); - smbios->len = entry.smbios_len; - smbios->count = entry.smbios_count; - smbios->version = SMBIOS_VERSION ( entry.major, entry.minor ); + smbios->address = phys_to_virt ( entry->smbios_address ); + smbios->len = entry->smbios_len; + smbios->count = entry->smbios_count; + smbios->version = SMBIOS_VERSION ( entry->major, entry->minor ); return 0; } @@ -69,26 +68,25 @@ static int bios_find_smbios2 ( struct smbios *smbios ) { * @ret rc Return status code */ static int bios_find_smbios3 ( struct smbios *smbios ) { - struct smbios3_entry entry; - int rc; + const struct smbios3_entry *entry; /* Scan through BIOS segment to find SMBIOS 64-bit entry point */ - if ( ( rc = find_smbios3_entry ( real_to_user ( BIOS_SEG, 0 ), 0x10000, - &entry ) ) != 0 ) - return rc; + entry = find_smbios3_entry ( real_to_virt ( BIOS_SEG, 0 ), 0x10000 ); + if ( ! entry ) + return -ENOENT; /* Check that address is accessible */ - if ( entry.smbios_address > ~( ( physaddr_t ) 0 ) ) { + if ( entry->smbios_address > ~( ( physaddr_t ) 0 ) ) { DBG ( "SMBIOS3 at %08llx is inaccessible\n", - ( ( unsigned long long ) entry.smbios_address ) ); + ( ( unsigned long long ) entry->smbios_address ) ); return -ENOTSUP; } /* Fill in entry point descriptor structure */ - smbios->address = phys_to_user ( entry.smbios_address ); - smbios->len = entry.smbios_len; + smbios->address = phys_to_virt ( entry->smbios_address ); + smbios->len = entry->smbios_len; smbios->count = 0; - smbios->version = SMBIOS_VERSION ( entry.major, entry.minor ); + smbios->version = SMBIOS_VERSION ( entry->major, entry->minor ); return 0; } diff --git a/src/arch/x86/interface/pcbios/biosint.c b/src/arch/x86/interface/pcbios/biosint.c index 667e9ed81..f5e54ede8 100644 --- a/src/arch/x86/interface/pcbios/biosint.c +++ b/src/arch/x86/interface/pcbios/biosint.c @@ -1,3 +1,4 @@ +#include <string.h> #include <errno.h> #include <realmode.h> #include <biosint.h> diff --git a/src/arch/x86/interface/pcbios/e820mangler.S b/src/arch/x86/interface/pcbios/e820mangler.S index ef5dc2754..b9e891dff 100644 --- a/src/arch/x86/interface/pcbios/e820mangler.S +++ b/src/arch/x86/interface/pcbios/e820mangler.S @@ -564,6 +564,8 @@ int15_88: int15: /* See if we want to intercept this call */ pushfw + cmpb $0, %cs:int15_intercept_flag + je 3f cmpw $0xe820, %ax jne 1f cmpl $SMAP, %edx @@ -587,3 +589,9 @@ int15: int15_vector: .long 0 .size int15_vector, . - int15_vector + + .section ".text16.data", "aw", @progbits + .globl int15_intercept_flag +int15_intercept_flag: + .byte 1 + .size int15_intercept_flag, . - int15_intercept_flag diff --git a/src/arch/x86/interface/pcbios/hidemem.c b/src/arch/x86/interface/pcbios/hidemem.c index 1a3022c5d..2b85459b6 100644 --- a/src/arch/x86/interface/pcbios/hidemem.c +++ b/src/arch/x86/interface/pcbios/hidemem.c @@ -22,6 +22,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +#include <string.h> #include <assert.h> #include <realmode.h> #include <biosint.h> @@ -29,7 +30,8 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <fakee820.h> #include <ipxe/init.h> #include <ipxe/io.h> -#include <ipxe/hidemem.h> +#include <ipxe/uheap.h> +#include <ipxe/memmap.h> /** Set to true if you want to test a fake E820 map */ #define FAKE_E820 0 @@ -72,13 +74,17 @@ extern void int15(); extern struct segoff __text16 ( int15_vector ); #define int15_vector __use_text16 ( int15_vector ) +/** INT 15 interception flag */ +extern uint8_t __text16 ( int15_intercept_flag ); +#define int15_intercept_flag __use_text16 ( int15_intercept_flag ) + /* The linker defines these symbols for us */ extern char _textdata[]; extern char _etextdata[]; -extern char _text16_memsz[]; -#define _text16_memsz ( ( size_t ) _text16_memsz ) -extern char _data16_memsz[]; -#define _data16_memsz ( ( size_t ) _data16_memsz ) +extern size_t ABS_SYMBOL ( _text16_memsz ); +#define _text16_memsz ABS_VALUE ( _text16_memsz ) +extern size_t ABS_SYMBOL ( _data16_memsz ); +#define _data16_memsz ABS_VALUE ( _data16_memsz ) /** * Hide region of memory from system memory map @@ -113,21 +119,43 @@ void hide_basemem ( void ) { } /** - * Hide umalloc() region + * Hide .text and .data + * + */ +void hide_textdata ( void ) { + hide_region ( &hidemem_textdata, virt_to_phys ( _textdata ), + virt_to_phys ( _etextdata ) ); +} + +/** + * Synchronise in-use regions with the externally visible system memory map * */ -void hide_umalloc ( physaddr_t start, physaddr_t end ) { - assert ( end <= virt_to_phys ( _textdata ) ); +static void int15_sync ( void ) { + physaddr_t start; + physaddr_t end; + + /* Besides our fixed base memory and textdata regions, we + * support hiding only a single in-use memory region (the + * umalloc region), which must be placed before the hidden + * textdata region (even if zero-length). + */ + start = uheap_start; + end = uheap_end; + if ( start == end ) + start = end = virt_to_phys ( _textdata ); hide_region ( &hidemem_umalloc, start, end ); } /** - * Hide .text and .data + * Set INT 15 interception flag * + * @v intercept Intercept INT 15 calls to modify memory map */ -void hide_textdata ( void ) { - hide_region ( &hidemem_textdata, virt_to_phys ( _textdata ), - virt_to_phys ( _etextdata ) ); +void int15_intercept ( int intercept ) { + + /* Set flag for INT 15 handler */ + int15_intercept_flag = intercept; } /** @@ -137,26 +165,25 @@ void hide_textdata ( void ) { * returned by the BIOS. */ static void hide_etherboot ( void ) { - struct memory_map memmap; unsigned int rm_ds_top; unsigned int rm_cs_top; unsigned int fbms; /* Dump memory map before mangling */ DBG ( "Hiding iPXE from system memory map\n" ); - get_memmap ( &memmap ); + memmap_dump_all ( 1 ); /* Hook in fake E820 map, if we're testing one */ if ( FAKE_E820 ) { DBG ( "Hooking in fake E820 map\n" ); fake_e820(); - get_memmap ( &memmap ); + memmap_dump_all ( 1 ); } /* Initialise the hidden regions */ hide_basemem(); - hide_umalloc ( virt_to_phys ( _textdata ), virt_to_phys ( _textdata ) ); hide_textdata(); + int15_sync(); /* Some really moronic BIOSes bring up the PXE stack via the * UNDI loader entry point and then don't bother to unload it @@ -183,7 +210,7 @@ static void hide_etherboot ( void ) { /* Dump memory map after mangling */ DBG ( "Hidden iPXE from system memory map\n" ); - get_memmap ( &memmap ); + memmap_dump_all ( 1 ); } /** @@ -193,7 +220,6 @@ static void hide_etherboot ( void ) { * possible. */ static void unhide_etherboot ( int flags __unused ) { - struct memory_map memmap; int rc; /* If we have more than one hooked interrupt at this point, it @@ -224,7 +250,7 @@ static void unhide_etherboot ( int flags __unused ) { /* Dump memory map after unhiding */ DBG ( "Unhidden iPXE from system memory map\n" ); - get_memmap ( &memmap ); + memmap_dump_all ( 1 ); } /** Hide Etherboot startup function */ @@ -233,3 +259,5 @@ struct startup_fn hide_etherboot_startup_fn __startup_fn ( STARTUP_EARLY ) = { .startup = hide_etherboot, .shutdown = unhide_etherboot, }; + +PROVIDE_MEMMAP ( int15, memmap_sync, int15_sync ); diff --git a/src/arch/x86/interface/pcbios/int13.c b/src/arch/x86/interface/pcbios/int13.c index 372d40ba3..3fb25d261 100644 --- a/src/arch/x86/interface/pcbios/int13.c +++ b/src/arch/x86/interface/pcbios/int13.c @@ -25,17 +25,18 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdint.h> #include <stdlib.h> +#include <string.h> #include <limits.h> #include <byteswap.h> #include <errno.h> #include <assert.h> #include <ipxe/blockdev.h> -#include <ipxe/io.h> #include <ipxe/acpi.h> #include <ipxe/sanboot.h> #include <ipxe/device.h> #include <ipxe/pci.h> #include <ipxe/eltorito.h> +#include <ipxe/memmap.h> #include <realmode.h> #include <bios.h> #include <biosint.h> @@ -181,8 +182,7 @@ static int int13_parse_eltorito ( struct san_device *sandev, void *scratch ) { int rc; /* Read boot record volume descriptor */ - if ( ( rc = sandev_read ( sandev, ELTORITO_LBA, 1, - virt_to_user ( boot ) ) ) != 0 ) { + if ( ( rc = sandev_read ( sandev, ELTORITO_LBA, 1, boot ) ) != 0 ) { DBGC ( sandev->drive, "INT13 drive %02x could not read El " "Torito boot record volume descriptor: %s\n", sandev->drive, strerror ( rc ) ); @@ -228,7 +228,7 @@ static int int13_guess_geometry_hdd ( struct san_device *sandev, void *scratch, int rc; /* Read partition table */ - if ( ( rc = sandev_read ( sandev, 0, 1, virt_to_user ( mbr ) ) ) != 0 ) { + if ( ( rc = sandev_read ( sandev, 0, 1, mbr ) ) != 0 ) { DBGC ( sandev->drive, "INT13 drive %02x could not read " "partition table to guess geometry: %s\n", sandev->drive, strerror ( rc ) ); @@ -517,12 +517,12 @@ static int int13_rw_sectors ( struct san_device *sandev, int ( * sandev_rw ) ( struct san_device *sandev, uint64_t lba, unsigned int count, - userptr_t buffer ) ) { + void *buffer ) ) { struct int13_data *int13 = sandev->priv; unsigned int cylinder, head, sector; unsigned long lba; unsigned int count; - userptr_t buffer; + void *buffer; int rc; /* Validate blocksize */ @@ -549,7 +549,7 @@ static int int13_rw_sectors ( struct san_device *sandev, lba = ( ( ( ( cylinder * int13->heads ) + head ) * int13->sectors_per_track ) + sector - 1 ); count = ix86->regs.al; - buffer = real_to_user ( ix86->segs.es, ix86->regs.bx ); + buffer = real_to_virt ( ix86->segs.es, ix86->regs.bx ); DBGC2 ( sandev->drive, "C/H/S %d/%d/%d = LBA %08lx <-> %04x:%04x " "(count %d)\n", cylinder, head, sector, lba, ix86->segs.es, @@ -710,12 +710,12 @@ static int int13_extended_rw ( struct san_device *sandev, int ( * sandev_rw ) ( struct san_device *sandev, uint64_t lba, unsigned int count, - userptr_t buffer ) ) { + void *buffer ) ) { struct int13_disk_address addr; uint8_t bufsize; uint64_t lba; unsigned long count; - userptr_t buffer; + void *buffer; int rc; /* Extended reads are not allowed on floppy drives. @@ -743,11 +743,11 @@ static int int13_extended_rw ( struct san_device *sandev, if ( ( addr.count == 0xff ) || ( ( addr.buffer.segment == 0xffff ) && ( addr.buffer.offset == 0xffff ) ) ) { - buffer = phys_to_user ( addr.buffer_phys ); + buffer = phys_to_virt ( addr.buffer_phys ); DBGC2 ( sandev->drive, "%08llx", ( ( unsigned long long ) addr.buffer_phys ) ); } else { - buffer = real_to_user ( addr.buffer.segment, + buffer = real_to_virt ( addr.buffer.segment, addr.buffer.offset ); DBGC2 ( sandev->drive, "%04x:%04x", addr.buffer.segment, addr.buffer.offset ); @@ -1058,7 +1058,7 @@ static int int13_cdrom_read_boot_catalog ( struct san_device *sandev, /* Read from boot catalog */ if ( ( rc = sandev_read ( sandev, start, command.count, - phys_to_user ( command.buffer ) ) ) != 0 ) { + phys_to_virt ( command.buffer ) ) ) != 0 ) { DBGC ( sandev->drive, "INT13 drive %02x could not read boot " "catalog: %s\n", sandev->drive, strerror ( rc ) ); return -INT13_STATUS_READ_ERROR; @@ -1455,8 +1455,8 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) { "catalog (status %04x)\n", drive, status ); return -EIO; } - copy_from_user ( &catalog, phys_to_user ( eltorito_cmd.buffer ), 0, - sizeof ( catalog ) ); + memcpy ( &catalog, phys_to_virt ( eltorito_cmd.buffer ), + sizeof ( catalog ) ); /* Sanity checks */ if ( catalog.valid.platform_id != ELTORITO_PLATFORM_X86 ) { @@ -1523,7 +1523,6 @@ static int int13_load_eltorito ( unsigned int drive, struct segoff *address ) { */ static int int13_boot ( unsigned int drive, struct san_boot_config *config __unused ) { - struct memory_map memmap; struct segoff address; int rc; @@ -1537,7 +1536,7 @@ static int int13_boot ( unsigned int drive, * many problems that turn out to be memory-map related that * it's worth doing. */ - get_memmap ( &memmap ); + memmap_dump_all ( 1 ); /* Jump to boot sector */ if ( ( rc = call_bootsector ( address.segment, address.offset, diff --git a/src/arch/x86/interface/pcbios/int13con.c b/src/arch/x86/interface/pcbios/int13con.c index 8106cd153..925228874 100644 --- a/src/arch/x86/interface/pcbios/int13con.c +++ b/src/arch/x86/interface/pcbios/int13con.c @@ -288,6 +288,7 @@ static void int13con_init ( void ) { * INT13 console initialisation function */ struct init_fn int13con_init_fn __init_fn ( INIT_CONSOLE ) = { + .name = "int13con", .initialise = int13con_init, }; diff --git a/src/arch/x86/interface/pcbios/memmap.c b/src/arch/x86/interface/pcbios/int15.c index daae382b8..73bdbbe4a 100644 --- a/src/arch/x86/interface/pcbios/memmap.c +++ b/src/arch/x86/interface/pcbios/int15.c @@ -24,11 +24,14 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdint.h> +#include <string.h> #include <errno.h> +#include <assert.h> #include <realmode.h> #include <bios.h> #include <memsizes.h> #include <ipxe/io.h> +#include <ipxe/memmap.h> /** * @file @@ -151,7 +154,7 @@ static unsigned int extmemsize_88 ( void ) { * @ret extmem Extended memory size, in kB * * Note that this is only an approximation; for an accurate picture, - * use the E820 memory map obtained via get_memmap(); + * use the E820 memory map obtained via memmap_describe(); */ unsigned int extmemsize ( void ) { unsigned int extmem_e801; @@ -166,12 +169,13 @@ unsigned int extmemsize ( void ) { /** * Get e820 memory map * - * @v memmap Memory map to fill in + * @v region Memory region of interest to be updated * @ret rc Return status code */ -static int meme820 ( struct memory_map *memmap ) { - struct memory_region *region = memmap->regions; - struct memory_region *prev_region = NULL; +static int meme820 ( struct memmap_region *region ) { + unsigned int count = 0; + uint64_t start = 0; + uint64_t len = 0; uint32_t next = 0; uint32_t smap; uint32_t size; @@ -225,13 +229,6 @@ static int meme820 ( struct memory_map *memmap ) { break; } - /* If first region is not RAM, assume map is invalid */ - if ( ( memmap->count == 0 ) && - ( e820buf.type != E820_TYPE_RAM ) ) { - DBG ( "INT 15,e820 failed, first entry not RAM\n" ); - return -EINVAL; - } - DBG ( "INT 15,e820 region [%llx,%llx) type %d", e820buf.start, ( e820buf.start + e820buf.len ), ( int ) e820buf.type ); @@ -258,27 +255,36 @@ static int meme820 ( struct memory_map *memmap ) { continue; } - region->start = e820buf.start; - region->end = e820buf.start + e820buf.len; - /* Check for adjacent regions and merge them */ - if ( prev_region && ( region->start == prev_region->end ) ) { - prev_region->end = region->end; + if ( e820buf.start == ( start + len ) ) { + len += e820buf.len; } else { - prev_region = region; - region++; - memmap->count++; + start = e820buf.start; + len = e820buf.len; } - if ( memmap->count >= ( sizeof ( memmap->regions ) / - sizeof ( memmap->regions[0] ) ) ) { - DBG ( "INT 15,e820 too many regions returned\n" ); - /* Not a fatal error; what we've got so far at - * least represents valid regions of memory, - * even if we couldn't get them all. - */ - break; + /* Sanity check: first region (base memory) should + * start at address zero. + */ + if ( ( count == 0 ) && ( start != 0 ) ) { + DBG ( "INT 15,e820 region 0 starts at %llx (expected " + "0); assuming insane\n", start ); + return -EINVAL; + } + + /* Sanity check: second region (extended memory) + * should start at address 0x100000. + */ + if ( ( count == 1 ) && ( start != 0x100000 ) ) { + DBG ( "INT 15,e820 region 1 starts at %llx (expected " + "100000); assuming insane\n", start ); + return -EINVAL; } + + /* Update region of interest */ + memmap_update ( region, start, len, MEMMAP_FL_MEMORY, "e820" ); + count++; + } while ( next != 0 ); /* Sanity checks. Some BIOSes report complete garbage via INT @@ -287,19 +293,9 @@ static int meme820 ( struct memory_map *memmap ) { * region (starting at 0) and at least one high memory region * (starting at 0x100000). */ - if ( memmap->count < 2 ) { + if ( count < 2 ) { DBG ( "INT 15,e820 returned only %d regions; assuming " - "insane\n", memmap->count ); - return -EINVAL; - } - if ( memmap->regions[0].start != 0 ) { - DBG ( "INT 15,e820 region 0 starts at %llx (expected 0); " - "assuming insane\n", memmap->regions[0].start ); - return -EINVAL; - } - if ( memmap->regions[1].start != 0x100000 ) { - DBG ( "INT 15,e820 region 1 starts at %llx (expected 100000); " - "assuming insane\n", memmap->regions[0].start ); + "insane\n", count ); return -EINVAL; } @@ -307,37 +303,52 @@ static int meme820 ( struct memory_map *memmap ) { } /** - * Get memory map + * Describe memory region from system memory map * - * @v memmap Memory map to fill in + * @v min Minimum address + * @v hide Hide in-use regions from the memory map + * @v region Region descriptor to fill in */ -void x86_get_memmap ( struct memory_map *memmap ) { - unsigned int basemem, extmem; +static void int15_describe ( uint64_t min, int hide, + struct memmap_region *region ) { + unsigned int basemem; + unsigned int extmem; + uint64_t inaccessible; int rc; - DBG ( "Fetching system memory map\n" ); + /* Initialise region */ + memmap_init ( min, region ); - /* Clear memory map */ - memset ( memmap, 0, sizeof ( *memmap ) ); + /* Mark addresses above 4GB as inaccessible: we have no way to + * access them either in a 32-bit build or in a 64-bit build + * (since the 64-bit build identity-maps only the 32-bit + * address space). + */ + inaccessible = ( 1ULL << 32 ); + memmap_update ( region, inaccessible, -inaccessible, + MEMMAP_FL_INACCESSIBLE, NULL ); - /* Get base and extended memory sizes */ - basemem = basememsize(); - DBG ( "FBMS base memory size %d kB [0,%x)\n", - basemem, ( basemem * 1024 ) ); - extmem = extmemsize(); - - /* Try INT 15,e820 first */ - if ( ( rc = meme820 ( memmap ) ) == 0 ) { + /* Enable/disable INT 15 interception as applicable */ + int15_intercept ( hide ); + + /* Try INT 15,e820 first, falling back to constructing a map + * from basemem and extmem sizes + */ + if ( ( rc = meme820 ( region ) ) == 0 ) { DBG ( "Obtained system memory map via INT 15,e820\n" ); - return; + } else { + basemem = basememsize(); + DBG ( "FBMS base memory size %d kB [0,%x)\n", + basemem, ( basemem * 1024 ) ); + extmem = extmemsize(); + memmap_update ( region, 0, ( basemem * 1024 ), + MEMMAP_FL_MEMORY, "basemem" ); + memmap_update ( region, 0x100000, ( extmem * 1024 ), + MEMMAP_FL_MEMORY, "extmem" ); } - /* Fall back to constructing a map from basemem and extmem sizes */ - DBG ( "INT 15,e820 failed; constructing map\n" ); - memmap->regions[0].end = ( basemem * 1024 ); - memmap->regions[1].start = 0x100000; - memmap->regions[1].end = 0x100000 + ( extmem * 1024 ); - memmap->count = 2; + /* Restore INT 15 interception */ + int15_intercept ( 1 ); } -PROVIDE_IOAPI ( x86, get_memmap, x86_get_memmap ); +PROVIDE_MEMMAP ( int15, memmap_describe, int15_describe ); diff --git a/src/arch/x86/interface/pcbios/memtop_umalloc.c b/src/arch/x86/interface/pcbios/memtop_umalloc.c deleted file mode 100644 index 1d3f40a1c..000000000 --- a/src/arch/x86/interface/pcbios/memtop_umalloc.c +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright (C) 2007 Michael Brown <mbrown@fensystems.co.uk>. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - * - * You can also choose to distribute this program under the terms of - * the Unmodified Binary Distribution Licence (as given in the file - * COPYING.UBDL), provided that you have satisfied its requirements. - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -/** - * @file - * - * External memory allocation - * - */ - -#include <limits.h> -#include <errno.h> -#include <ipxe/uaccess.h> -#include <ipxe/hidemem.h> -#include <ipxe/io.h> -#include <ipxe/memblock.h> -#include <ipxe/umalloc.h> - -/** Maximum usable address for external allocated memory */ -#define EM_MAX_ADDRESS 0xffffffffUL - -/** Alignment of external allocated memory */ -#define EM_ALIGN ( 4 * 1024 ) - -/** Equivalent of NOWHERE for user pointers */ -#define UNOWHERE ( ~UNULL ) - -/** An external memory block */ -struct external_memory { - /** Size of this memory block (excluding this header) */ - size_t size; - /** Block is currently in use */ - int used; -}; - -/** Top of heap */ -static userptr_t top = UNULL; - -/** Bottom of heap (current lowest allocated block) */ -static userptr_t bottom = UNULL; - -/** Remaining space on heap */ -static size_t heap_size; - -/** - * Find largest usable memory region - * - * @ret start Start of region - * @ret len Length of region - */ -size_t largest_memblock ( userptr_t *start ) { - struct memory_map memmap; - struct memory_region *region; - physaddr_t max = EM_MAX_ADDRESS; - physaddr_t region_start; - physaddr_t region_end; - size_t region_len; - unsigned int i; - size_t len = 0; - - /* Avoid returning uninitialised data on error */ - *start = UNULL; - - /* Scan through all memory regions */ - get_memmap ( &memmap ); - for ( i = 0 ; i < memmap.count ; i++ ) { - region = &memmap.regions[i]; - DBG ( "Considering [%llx,%llx)\n", region->start, region->end ); - - /* Truncate block to maximum physical address */ - if ( region->start > max ) { - DBG ( "...starts after maximum address %lx\n", max ); - continue; - } - region_start = region->start; - if ( region->end > max ) { - DBG ( "...end truncated to maximum address %lx\n", max); - region_end = 0; /* =max, given the wraparound */ - } else { - region_end = region->end; - } - region_len = ( region_end - region_start ); - - /* Use largest block */ - if ( region_len > len ) { - DBG ( "...new best block found\n" ); - *start = phys_to_user ( region_start ); - len = region_len; - } - } - - return len; -} - -/** - * Initialise external heap - * - */ -static void init_eheap ( void ) { - userptr_t base; - - heap_size = largest_memblock ( &base ); - bottom = top = userptr_add ( base, heap_size ); - DBG ( "External heap grows downwards from %lx (size %zx)\n", - user_to_phys ( top, 0 ), heap_size ); -} - -/** - * Collect free blocks - * - */ -static void ecollect_free ( void ) { - struct external_memory extmem; - size_t len; - - /* Walk the free list and collect empty blocks */ - while ( bottom != top ) { - copy_from_user ( &extmem, bottom, -sizeof ( extmem ), - sizeof ( extmem ) ); - if ( extmem.used ) - break; - DBG ( "EXTMEM freeing [%lx,%lx)\n", user_to_phys ( bottom, 0 ), - user_to_phys ( bottom, extmem.size ) ); - len = ( extmem.size + sizeof ( extmem ) ); - bottom = userptr_add ( bottom, len ); - heap_size += len; - } -} - -/** - * Reallocate external memory - * - * @v old_ptr Memory previously allocated by umalloc(), or UNULL - * @v new_size Requested size - * @ret new_ptr Allocated memory, or UNULL - * - * Calling realloc() with a new size of zero is a valid way to free a - * memory block. - */ -static userptr_t memtop_urealloc ( userptr_t ptr, size_t new_size ) { - struct external_memory extmem; - userptr_t new = ptr; - size_t align; - - /* (Re)initialise external memory allocator if necessary */ - if ( bottom == top ) - init_eheap(); - - /* Get block properties into extmem */ - if ( ptr && ( ptr != UNOWHERE ) ) { - /* Determine old size */ - copy_from_user ( &extmem, ptr, -sizeof ( extmem ), - sizeof ( extmem ) ); - } else { - /* Create a zero-length block */ - if ( heap_size < sizeof ( extmem ) ) { - DBG ( "EXTMEM out of space\n" ); - return UNULL; - } - ptr = bottom = userptr_add ( bottom, -sizeof ( extmem ) ); - heap_size -= sizeof ( extmem ); - DBG ( "EXTMEM allocating [%lx,%lx)\n", - user_to_phys ( ptr, 0 ), user_to_phys ( ptr, 0 ) ); - extmem.size = 0; - } - extmem.used = ( new_size > 0 ); - - /* Expand/shrink block if possible */ - if ( ptr == bottom ) { - /* Update block */ - new = userptr_add ( ptr, - ( new_size - extmem.size ) ); - align = ( user_to_phys ( new, 0 ) & ( EM_ALIGN - 1 ) ); - new_size += align; - new = userptr_add ( new, -align ); - if ( new_size > ( heap_size + extmem.size ) ) { - DBG ( "EXTMEM out of space\n" ); - return UNULL; - } - DBG ( "EXTMEM expanding [%lx,%lx) to [%lx,%lx)\n", - user_to_phys ( ptr, 0 ), - user_to_phys ( ptr, extmem.size ), - user_to_phys ( new, 0 ), - user_to_phys ( new, new_size )); - memmove_user ( new, 0, ptr, 0, ( ( extmem.size < new_size ) ? - extmem.size : new_size ) ); - bottom = new; - heap_size -= ( new_size - extmem.size ); - extmem.size = new_size; - } else { - /* Cannot expand; can only pretend to shrink */ - if ( new_size > extmem.size ) { - /* Refuse to expand */ - DBG ( "EXTMEM cannot expand [%lx,%lx)\n", - user_to_phys ( ptr, 0 ), - user_to_phys ( ptr, extmem.size ) ); - return UNULL; - } - } - - /* Write back block properties */ - copy_to_user ( new, -sizeof ( extmem ), &extmem, - sizeof ( extmem ) ); - - /* Collect any free blocks and update hidden memory region */ - ecollect_free(); - hide_umalloc ( user_to_phys ( bottom, ( ( bottom == top ) ? - 0 : -sizeof ( extmem ) ) ), - user_to_phys ( top, 0 ) ); - - return ( new_size ? new : UNOWHERE ); -} - -PROVIDE_UMALLOC ( memtop, urealloc, memtop_urealloc ); diff --git a/src/arch/x86/interface/pcbios/pcibios.c b/src/arch/x86/interface/pcbios/pcibios.c index 7b7a769e3..6b88ee907 100644 --- a/src/arch/x86/interface/pcbios/pcibios.c +++ b/src/arch/x86/interface/pcbios/pcibios.c @@ -120,6 +120,7 @@ int pcibios_write ( struct pci_device *pci, uint32_t command, uint32_t value ){ return ( status >> 8 ); } +PROVIDE_PCIAPI_INLINE ( pcbios, pci_can_probe ); PROVIDE_PCIAPI ( pcbios, pci_discover, pcibios_discover ); PROVIDE_PCIAPI_INLINE ( pcbios, pci_read_config_byte ); PROVIDE_PCIAPI_INLINE ( pcbios, pci_read_config_word ); @@ -128,5 +129,4 @@ PROVIDE_PCIAPI_INLINE ( pcbios, pci_write_config_byte ); PROVIDE_PCIAPI_INLINE ( pcbios, pci_write_config_word ); PROVIDE_PCIAPI_INLINE ( pcbios, pci_write_config_dword ); PROVIDE_PCIAPI_INLINE ( pcbios, pci_ioremap ); - -struct pci_api pcibios_api = PCIAPI_RUNTIME ( pcbios ); +PROVIDE_PCIAPI_RUNTIME ( pcbios, PCIAPI_PRIORITY_PCBIOS ); diff --git a/src/arch/x86/interface/pcbios/pcicloud.c b/src/arch/x86/interface/pcbios/pcicloud.c deleted file mode 100644 index 98ba38b31..000000000 --- a/src/arch/x86/interface/pcbios/pcicloud.c +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright (C) 2022 Michael Brown <mbrown@fensystems.co.uk>. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License as - * published by the Free Software Foundation; either version 2 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - * 02110-1301, USA. - * - * You can also choose to distribute this program under the terms of - * the Unmodified Binary Distribution Licence (as given in the file - * COPYING.UBDL), provided that you have satisfied its requirements. - */ - -FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); - -#include <stdint.h> -#include <ipxe/init.h> -#include <ipxe/pci.h> -#include <ipxe/ecam.h> -#include <ipxe/pcibios.h> -#include <ipxe/pcidirect.h> -#include <ipxe/pcicloud.h> - -/** @file - * - * Cloud VM PCI configuration space access - * - */ - -/** Selected PCI configuration space access API */ -static struct pci_api *pcicloud = &ecam_api; - -/** - * Find next PCI bus:dev.fn address range in system - * - * @v busdevfn Starting PCI bus:dev.fn address - * @v range PCI bus:dev.fn address range to fill in - */ -static void pcicloud_discover ( uint32_t busdevfn, struct pci_range *range ) { - - pcicloud->pci_discover ( busdevfn, range ); -} - -/** - * Read byte from PCI configuration space - * - * @v pci PCI device - * @v where Location within PCI configuration space - * @v value Value read - * @ret rc Return status code - */ -static int pcicloud_read_config_byte ( struct pci_device *pci, - unsigned int where, uint8_t *value ) { - - return pcicloud->pci_read_config_byte ( pci, where, value ); -} - -/** - * Read 16-bit word from PCI configuration space - * - * @v pci PCI device - * @v where Location within PCI configuration space - * @v value Value read - * @ret rc Return status code - */ -static int pcicloud_read_config_word ( struct pci_device *pci, - unsigned int where, uint16_t *value ) { - - return pcicloud->pci_read_config_word ( pci, where, value ); -} - -/** - * Read 32-bit dword from PCI configuration space - * - * @v pci PCI device - * @v where Location within PCI configuration space - * @v value Value read - * @ret rc Return status code - */ -static int pcicloud_read_config_dword ( struct pci_device *pci, - unsigned int where, uint32_t *value ) { - - return pcicloud->pci_read_config_dword ( pci, where, value ); -} - -/** - * Write byte to PCI configuration space - * - * @v pci PCI device - * @v where Location within PCI configuration space - * @v value Value to be written - * @ret rc Return status code - */ -static int pcicloud_write_config_byte ( struct pci_device *pci, - unsigned int where, uint8_t value ) { - - return pcicloud->pci_write_config_byte ( pci, where, value ); -} - -/** - * Write 16-bit word to PCI configuration space - * - * @v pci PCI device - * @v where Location within PCI configuration space - * @v value Value to be written - * @ret rc Return status code - */ -static int pcicloud_write_config_word ( struct pci_device *pci, - unsigned int where, uint16_t value ) { - - return pcicloud->pci_write_config_word ( pci, where, value ); -} - -/** - * Write 32-bit dword to PCI configuration space - * - * @v pci PCI device - * @v where Location within PCI configuration space - * @v value Value to be written - * @ret rc Return status code - */ -static int pcicloud_write_config_dword ( struct pci_device *pci, - unsigned int where, uint32_t value ) { - - return pcicloud->pci_write_config_dword ( pci, where, value ); -} - -/** - * Map PCI bus address as an I/O address - * - * @v bus_addr PCI bus address - * @v len Length of region - * @ret io_addr I/O address, or NULL on error - */ -static void * pcicloud_ioremap ( struct pci_device *pci, - unsigned long bus_addr, size_t len ) { - - return pcicloud->pci_ioremap ( pci, bus_addr, len ); -} - -PROVIDE_PCIAPI ( cloud, pci_discover, pcicloud_discover ); -PROVIDE_PCIAPI ( cloud, pci_read_config_byte, pcicloud_read_config_byte ); -PROVIDE_PCIAPI ( cloud, pci_read_config_word, pcicloud_read_config_word ); -PROVIDE_PCIAPI ( cloud, pci_read_config_dword, pcicloud_read_config_dword ); -PROVIDE_PCIAPI ( cloud, pci_write_config_byte, pcicloud_write_config_byte ); -PROVIDE_PCIAPI ( cloud, pci_write_config_word, pcicloud_write_config_word ); -PROVIDE_PCIAPI ( cloud, pci_write_config_dword, pcicloud_write_config_dword ); -PROVIDE_PCIAPI ( cloud, pci_ioremap, pcicloud_ioremap ); - -/** - * Initialise cloud VM PCI configuration space access - * - */ -static void pcicloud_init ( void ) { - static struct pci_api *apis[] = { - &ecam_api, &pcibios_api, &pcidirect_api - }; - struct pci_device pci; - uint32_t busdevfn; - unsigned int i; - int rc; - - /* Select first API that successfully discovers a PCI device */ - for ( i = 0 ; i < ( sizeof ( apis ) / sizeof ( apis[0] ) ) ; i++ ) { - pcicloud = apis[i]; - busdevfn = 0; - if ( ( rc = pci_find_next ( &pci, &busdevfn ) ) == 0 ) { - DBGC ( pcicloud, "PCICLOUD selected %s API (found " - PCI_FMT ")\n", pcicloud->name, - PCI_ARGS ( &pci ) ); - return; - } - } - - /* Fall back to using final attempted API if no devices found */ - pcicloud = apis[ i - 1 ]; - DBGC ( pcicloud, "PCICLOUD selected %s API (nothing detected)\n", - pcicloud->name ); -} - -/** Cloud VM PCI configuration space access initialisation function */ -struct init_fn pcicloud_init_fn __init_fn ( INIT_EARLY ) = { - .initialise = pcicloud_init, -}; diff --git a/src/arch/x86/interface/pcbios/rsdp.c b/src/arch/x86/interface/pcbios/rsdp.c index 3c67b7525..6913be552 100644 --- a/src/arch/x86/interface/pcbios/rsdp.c +++ b/src/arch/x86/interface/pcbios/rsdp.c @@ -31,6 +31,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); */ #include <stdint.h> +#include <string.h> #include <realmode.h> #include <bios.h> #include <ipxe/acpi.h> @@ -53,50 +54,51 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * * @v start Start address to search * @v len Length to search - * @ret rsdt ACPI root system description table, or UNULL + * @ret rsdt ACPI root system description table, or NULL */ -static userptr_t rsdp_find_rsdt_range ( userptr_t start, size_t len ) { +static const struct acpi_rsdt * rsdp_find_rsdt_range ( const void *start, + size_t len ) { static const char signature[8] = RSDP_SIGNATURE; - struct acpi_rsdp rsdp; - userptr_t rsdt; + const struct acpi_rsdp *rsdp; + const struct acpi_rsdt *rsdt; size_t offset; uint8_t sum; unsigned int i; /* Search for RSDP */ - for ( offset = 0 ; ( ( offset + sizeof ( rsdp ) ) < len ) ; + for ( offset = 0 ; ( ( offset + sizeof ( *rsdp ) ) < len ) ; offset += RSDP_STRIDE ) { /* Check signature and checksum */ - copy_from_user ( &rsdp, start, offset, sizeof ( rsdp ) ); - if ( memcmp ( rsdp.signature, signature, + rsdp = ( start + offset ); + if ( memcmp ( rsdp->signature, signature, sizeof ( signature ) ) != 0 ) continue; - for ( sum = 0, i = 0 ; i < sizeof ( rsdp ) ; i++ ) - sum += *( ( ( uint8_t * ) &rsdp ) + i ); + for ( sum = 0, i = 0 ; i < sizeof ( *rsdp ) ; i++ ) + sum += *( ( ( uint8_t * ) rsdp ) + i ); if ( sum != 0 ) continue; /* Extract RSDT */ - rsdt = phys_to_user ( le32_to_cpu ( rsdp.rsdt ) ); + rsdt = phys_to_virt ( le32_to_cpu ( rsdp->rsdt ) ); DBGC ( rsdt, "RSDT %#08lx found via RSDP %#08lx\n", - user_to_phys ( rsdt, 0 ), - user_to_phys ( start, offset ) ); + virt_to_phys ( rsdt ), + ( virt_to_phys ( start ) + offset ) ); return rsdt; } - return UNULL; + return NULL; } /** * Locate ACPI root system description table * - * @ret rsdt ACPI root system description table, or UNULL + * @ret rsdt ACPI root system description table, or NULL */ -static userptr_t rsdp_find_rsdt ( void ) { - static userptr_t rsdt; +static const struct acpi_rsdt * rsdp_find_rsdt ( void ) { + static const struct acpi_rsdt *rsdt; + const void *ebda; uint16_t ebda_seg; - userptr_t ebda; size_t ebda_len; /* Return existing RSDT if already found */ @@ -106,7 +108,7 @@ static userptr_t rsdp_find_rsdt ( void ) { /* Search EBDA */ get_real ( ebda_seg, BDA_SEG, BDA_EBDA ); if ( ebda_seg < RSDP_EBDA_END_SEG ) { - ebda = real_to_user ( ebda_seg, 0 ); + ebda = real_to_virt ( ebda_seg, 0 ); ebda_len = ( ( RSDP_EBDA_END_SEG - ebda_seg ) * 16 ); rsdt = rsdp_find_rsdt_range ( ebda, ebda_len ); if ( rsdt ) @@ -114,12 +116,12 @@ static userptr_t rsdp_find_rsdt ( void ) { } /* Search fixed BIOS area */ - rsdt = rsdp_find_rsdt_range ( phys_to_user ( RSDP_BIOS_START ), + rsdt = rsdp_find_rsdt_range ( phys_to_virt ( RSDP_BIOS_START ), RSDP_BIOS_LEN ); if ( rsdt ) return rsdt; - return UNULL; + return NULL; } PROVIDE_ACPI ( rsdp, acpi_find_rsdt, rsdp_find_rsdt ); diff --git a/src/arch/x86/interface/pcbios/rtc_entropy.c b/src/arch/x86/interface/pcbios/rtc_entropy.c index 8f47ff6b8..7c98019b6 100644 --- a/src/arch/x86/interface/pcbios/rtc_entropy.c +++ b/src/arch/x86/interface/pcbios/rtc_entropy.c @@ -53,6 +53,12 @@ extern void rtc_isr ( void ); /** Previous RTC interrupt handler */ static struct segoff rtc_old_handler; +/** Previous RTC interrupt enabled state */ +static uint8_t rtc_irq_enabled; + +/** Previous RTC periodic interrupt enabled state */ +static uint8_t rtc_int_enabled; + /** Flag set by RTC interrupt handler */ extern volatile uint8_t __text16 ( rtc_flag ); #define rtc_flag __use_text16 ( rtc_flag ) @@ -107,8 +113,9 @@ static void rtc_unhook_isr ( void ) { /** * Enable RTC interrupts * + * @ret enabled Periodic interrupt was previously enabled */ -static void rtc_enable_int ( void ) { +static int rtc_enable_int ( void ) { uint8_t status_b; /* Clear any stale pending interrupts via status register C */ @@ -124,6 +131,9 @@ static void rtc_enable_int ( void ) { /* Re-enable NMI and reset to default address */ outb ( CMOS_DEFAULT_ADDRESS, CMOS_ADDRESS ); inb ( CMOS_DATA ); /* Discard; may be needed on some platforms */ + + /* Return previous state */ + return ( status_b & RTC_STATUS_B_PIE ); } /** @@ -198,8 +208,11 @@ static int rtc_entropy_enable ( void ) { /* Hook ISR and enable RTC interrupts */ rtc_hook_isr(); - enable_irq ( RTC_IRQ ); - rtc_enable_int(); + rtc_irq_enabled = enable_irq ( RTC_IRQ ); + rtc_int_enabled = rtc_enable_int(); + DBGC ( &rtc_flag, "RTC had IRQ%d %sabled, interrupt %sabled\n", + RTC_IRQ, ( rtc_irq_enabled ? "en" : "dis" ), + ( rtc_int_enabled ? "en" : "dis" ) ); /* Check that RTC interrupts are working */ if ( ( rc = rtc_entropy_check() ) != 0 ) @@ -223,8 +236,10 @@ static int rtc_entropy_enable ( void ) { return 0; err_check: - rtc_disable_int(); - disable_irq ( RTC_IRQ ); + if ( ! rtc_int_enabled ) + rtc_disable_int(); + if ( ! rtc_irq_enabled ) + disable_irq ( RTC_IRQ ); rtc_unhook_isr(); err_no_tsc: return rc; @@ -236,9 +251,11 @@ static int rtc_entropy_enable ( void ) { */ static void rtc_entropy_disable ( void ) { - /* Disable RTC interrupts and unhook ISR */ - rtc_disable_int(); - disable_irq ( RTC_IRQ ); + /* Restore RTC interrupt state and unhook ISR */ + if ( ! rtc_int_enabled ) + rtc_disable_int(); + if ( ! rtc_irq_enabled ) + disable_irq ( RTC_IRQ ); rtc_unhook_isr(); } diff --git a/src/arch/x86/interface/pcbios/vesafb.c b/src/arch/x86/interface/pcbios/vesafb.c index 3ca15271a..48394bdd9 100644 --- a/src/arch/x86/interface/pcbios/vesafb.c +++ b/src/arch/x86/interface/pcbios/vesafb.c @@ -103,7 +103,7 @@ struct vesafb { /** Font definition */ struct fbcon_font font; /** Character glyphs */ - struct segoff glyphs; + const uint8_t *glyphs; /** Saved VGA mode */ uint8_t saved_mode; }; @@ -140,11 +140,10 @@ static int vesafb_rc ( unsigned int status ) { * Get character glyph * * @v character Unicode character - * @v glyph Character glyph to fill in + * @ret glyph Character glyph */ -static void vesafb_glyph ( unsigned int character, uint8_t *glyph ) { +static const uint8_t * vesafb_glyph ( unsigned int character ) { unsigned int index; - size_t offset; /* Identify glyph */ if ( character < VESAFB_ASCII ) { @@ -287,10 +286,8 @@ static void vesafb_glyph ( unsigned int character, uint8_t *glyph ) { } } - /* Copy glyph from BIOS font table */ - offset = ( index * VESAFB_CHAR_HEIGHT ); - copy_from_real ( glyph, vesafb.glyphs.segment, - ( vesafb.glyphs.offset + offset ), VESAFB_CHAR_HEIGHT); + /* Return glyph in BIOS font table */ + return &vesafb.glyphs[ index * VESAFB_CHAR_HEIGHT ]; } /** @@ -298,6 +295,7 @@ static void vesafb_glyph ( unsigned int character, uint8_t *glyph ) { * */ static void vesafb_font ( void ) { + struct segoff glyphs; /* Get font information * @@ -318,12 +316,13 @@ static void vesafb_font ( void ) { "movw %%es, %%cx\n\t" "movw %%bp, %%dx\n\t" "popw %%bp\n\t" /* gcc bug */ ) - : "=c" ( vesafb.glyphs.segment ), - "=d" ( vesafb.glyphs.offset ) + : "=c" ( glyphs.segment ), + "=d" ( glyphs.offset ) : "a" ( VBE_GET_FONT ), "b" ( VESAFB_FONT ) ); DBGC ( &vbe_buf, "VESAFB has font %04x at %04x:%04x\n", - VESAFB_FONT, vesafb.glyphs.segment, vesafb.glyphs.offset ); + VESAFB_FONT, glyphs.segment, glyphs.offset ); + vesafb.glyphs = real_to_virt ( glyphs.segment, glyphs.offset ); vesafb.font.height = VESAFB_CHAR_HEIGHT; vesafb.font.glyph = vesafb_glyph; } @@ -338,8 +337,8 @@ static void vesafb_font ( void ) { */ static int vesafb_mode_list ( uint16_t **mode_numbers ) { struct vbe_controller_info *controller = &vbe_buf.controller; - userptr_t video_mode_ptr; - uint16_t mode_number; + const uint16_t *video_mode_ptr; + const uint16_t *mode_number; uint16_t status; size_t len; int rc; @@ -375,20 +374,18 @@ static int vesafb_mode_list ( uint16_t **mode_numbers ) { controller->video_mode_ptr.offset ); /* Calculate length of mode list */ - video_mode_ptr = real_to_user ( controller->video_mode_ptr.segment, + video_mode_ptr = real_to_virt ( controller->video_mode_ptr.segment, controller->video_mode_ptr.offset ); - len = 0; - do { - copy_from_user ( &mode_number, video_mode_ptr, len, - sizeof ( mode_number ) ); - len += sizeof ( mode_number ); - } while ( mode_number != VBE_MODE_END ); + mode_number = video_mode_ptr; + while ( *(mode_number++) != VBE_MODE_END ) {} + len = ( ( ( const void * ) mode_number ) - + ( ( const void * ) video_mode_ptr ) ); /* Allocate and fill mode list */ *mode_numbers = malloc ( len ); if ( ! *mode_numbers ) return -ENOMEM; - copy_from_user ( *mode_numbers, video_mode_ptr, 0, len ); + memcpy ( *mode_numbers, video_mode_ptr, len ); return 0; } @@ -607,7 +604,7 @@ static int vesafb_init ( struct console_configuration *config ) { } /* Initialise frame buffer console */ - if ( ( rc = fbcon_init ( &vesafb.fbcon, phys_to_user ( vesafb.start ), + if ( ( rc = fbcon_init ( &vesafb.fbcon, phys_to_virt ( vesafb.start ), &vesafb.pixel, &vesafb.map, &vesafb.font, config ) ) != 0 ) goto err_fbcon_init; diff --git a/src/arch/x86/interface/pxe/pxe_call.c b/src/arch/x86/interface/pxe/pxe_call.c index 0e8d5c5a8..9a6a20dd3 100644 --- a/src/arch/x86/interface/pxe/pxe_call.c +++ b/src/arch/x86/interface/pxe/pxe_call.c @@ -55,12 +55,12 @@ extern void pxe_int_1a ( void ); static int int_1a_hooked = 0; /** Real-mode code segment size */ -extern char _text16_memsz[]; -#define _text16_memsz ( ( size_t ) _text16_memsz ) +extern size_t ABS_SYMBOL ( _text16_memsz ); +#define _text16_memsz ABS_VALUE ( _text16_memsz ) /** Real-mode data segment size */ -extern char _data16_memsz[]; -#define _data16_memsz ( ( size_t ) _data16_memsz ) +extern size_t ABS_SYMBOL (_data16_memsz ); +#define _data16_memsz ABS_VALUE ( _data16_memsz ) /** PXENV_UNDI_TRANSMIT API call profiler */ static struct profiler pxe_api_tx_profiler __profiler = @@ -144,10 +144,10 @@ static struct profiler * pxe_api_profiler ( unsigned int opcode ) { */ __asmcall void pxe_api_call ( struct i386_all_regs *ix86 ) { uint16_t opcode = ix86->regs.bx; - userptr_t uparams = real_to_user ( ix86->segs.es, ix86->regs.di ); struct profiler *profiler = pxe_api_profiler ( opcode ); + union u_PXENV_ANY *params = + real_to_virt ( ix86->segs.es, ix86->regs.di ); struct pxe_api_call *call; - union u_PXENV_ANY params; PXENV_EXIT_t ret; /* Start profiling */ @@ -160,17 +160,13 @@ __asmcall void pxe_api_call ( struct i386_all_regs *ix86 ) { call = &pxenv_unknown_api; } - /* Copy parameter block from caller */ - copy_from_user ( ¶ms, uparams, 0, call->params_len ); - /* Set default status in case child routine fails to do so */ - params.Status = PXENV_STATUS_FAILURE; + params->Status = PXENV_STATUS_FAILURE; /* Hand off to relevant API routine */ - ret = call->entry ( ¶ms ); + ret = call->entry ( params ); - /* Copy modified parameter block back to caller and return */ - copy_to_user ( uparams, 0, ¶ms, call->params_len ); + /* Return exit code in %ax */ ix86->regs.ax = ret; /* Stop profiling, if applicable */ @@ -195,24 +191,20 @@ int pxe_api_call_weak ( struct i386_all_regs *ix86 ) { * @ret ax PXE exit code */ __asmcall void pxe_loader_call ( struct i386_all_regs *ix86 ) { - userptr_t uparams = real_to_user ( ix86->segs.es, ix86->regs.di ); - struct s_UNDI_LOADER params; + struct s_UNDI_LOADER *params = + real_to_virt ( ix86->segs.es, ix86->regs.di ); PXENV_EXIT_t ret; - /* Copy parameter block from caller */ - copy_from_user ( ¶ms, uparams, 0, sizeof ( params ) ); - /* Fill in ROM segment address */ ppxe.UNDIROMID.segment = ix86->segs.ds; /* Set default status in case child routine fails to do so */ - params.Status = PXENV_STATUS_FAILURE; + params->Status = PXENV_STATUS_FAILURE; /* Call UNDI loader */ - ret = undi_loader ( ¶ms ); + ret = undi_loader ( params ); - /* Copy modified parameter block back to caller and return */ - copy_to_user ( uparams, 0, ¶ms, sizeof ( params ) ); + /* Return exit code in %ax */ ix86->regs.ax = ret; } @@ -265,6 +257,7 @@ static void pxe_init_structures ( void ) { /** PXE structure initialiser */ struct init_fn pxe_init_fn __init_fn ( INIT_NORMAL ) = { + .name = "pxe", .initialise = pxe_init_structures, }; diff --git a/src/arch/x86/interface/pxe/pxe_file.c b/src/arch/x86/interface/pxe/pxe_file.c index 456ffb5fd..997667ccf 100644 --- a/src/arch/x86/interface/pxe/pxe_file.c +++ b/src/arch/x86/interface/pxe/pxe_file.c @@ -8,7 +8,6 @@ #include <stdio.h> #include <errno.h> #include <byteswap.h> -#include <ipxe/uaccess.h> #include <ipxe/posix_io.h> #include <ipxe/features.h> #include <pxe.h> @@ -53,30 +52,20 @@ FEATURE ( FEATURE_MISC, "PXEXT", DHCP_EB_FEATURE_PXE_EXT, 2 ); * */ static PXENV_EXIT_t pxenv_file_open ( struct s_PXENV_FILE_OPEN *file_open ) { - userptr_t filename; - size_t filename_len; + const char *filename; int fd; DBG ( "PXENV_FILE_OPEN" ); - /* Copy name from external program, and open it */ - filename = real_to_user ( file_open->FileName.segment, - file_open->FileName.offset ); - filename_len = strlen_user ( filename, 0 ); - { - char uri_string[ filename_len + 1 ]; - - copy_from_user ( uri_string, filename, 0, - sizeof ( uri_string ) ); - DBG ( " %s", uri_string ); - fd = open ( uri_string ); - } - + /* Open specified filename */ + filename = real_to_virt ( file_open->FileName.segment, + file_open->FileName.offset ); + DBG ( " %s", filename ); + fd = open ( filename ); if ( fd < 0 ) { file_open->Status = PXENV_STATUS ( fd ); return PXENV_EXIT_FAILURE; } - DBG ( " as file %d", fd ); file_open->FileHandle = fd; @@ -148,17 +137,17 @@ pxenv_file_select ( struct s_PXENV_FILE_SELECT *file_select ) { * */ static PXENV_EXIT_t pxenv_file_read ( struct s_PXENV_FILE_READ *file_read ) { - userptr_t buffer; + void *buffer; ssize_t len; DBG ( "PXENV_FILE_READ %d to %04x:%04x+%04x", file_read->FileHandle, file_read->Buffer.segment, file_read->Buffer.offset, file_read->BufferSize ); - buffer = real_to_user ( file_read->Buffer.segment, + buffer = real_to_virt ( file_read->Buffer.segment, file_read->Buffer.offset ); - if ( ( len = read_user ( file_read->FileHandle, buffer, 0, - file_read->BufferSize ) ) < 0 ) { + if ( ( len = read ( file_read->FileHandle, buffer, + file_read->BufferSize ) ) < 0 ) { file_read->Status = PXENV_STATUS ( len ); return PXENV_EXIT_FAILURE; } @@ -210,27 +199,18 @@ pxenv_get_file_size ( struct s_PXENV_GET_FILE_SIZE *get_file_size ) { * */ static PXENV_EXIT_t pxenv_file_exec ( struct s_PXENV_FILE_EXEC *file_exec ) { - userptr_t command; - size_t command_len; + const char *command; int rc; DBG ( "PXENV_FILE_EXEC" ); - /* Copy name from external program, and exec it */ - command = real_to_user ( file_exec->Command.segment, + /* Execute specified command */ + command = real_to_virt ( file_exec->Command.segment, file_exec->Command.offset ); - command_len = strlen_user ( command, 0 ); - { - char command_string[ command_len + 1 ]; - - copy_from_user ( command_string, command, 0, - sizeof ( command_string ) ); - DBG ( " %s", command_string ); - - if ( ( rc = system ( command_string ) ) != 0 ) { - file_exec->Status = PXENV_STATUS ( rc ); - return PXENV_EXIT_FAILURE; - } + DBG ( " %s", command ); + if ( ( rc = system ( command ) ) != 0 ) { + file_exec->Status = PXENV_STATUS ( rc ); + return PXENV_EXIT_FAILURE; } file_exec->Status = PXENV_STATUS_SUCCESS; @@ -251,23 +231,22 @@ static PXENV_EXIT_t pxenv_file_exec ( struct s_PXENV_FILE_EXEC *file_exec ) { */ static PXENV_EXIT_t pxenv_file_cmdline ( struct s_PXENV_FILE_CMDLINE *file_cmdline ) { - userptr_t buffer; - size_t max_len; + char *buffer; size_t len; DBG ( "PXENV_FILE_CMDLINE to %04x:%04x+%04x \"%s\"\n", file_cmdline->Buffer.segment, file_cmdline->Buffer.offset, file_cmdline->BufferSize, pxe_cmdline ); - buffer = real_to_user ( file_cmdline->Buffer.segment, + buffer = real_to_virt ( file_cmdline->Buffer.segment, file_cmdline->Buffer.offset ); len = file_cmdline->BufferSize; - max_len = ( pxe_cmdline ? - ( strlen ( pxe_cmdline ) + 1 /* NUL */ ) : 0 ); - if ( len > max_len ) - len = max_len; - copy_to_user ( buffer, 0, pxe_cmdline, len ); - file_cmdline->BufferSize = max_len; + if ( pxe_cmdline ) { + len = snprintf ( buffer, len, "%s", pxe_cmdline ); + file_cmdline->BufferSize = ( len + 1 /* NUL */ ); + } else { + file_cmdline->BufferSize = 0; + } file_cmdline->Status = PXENV_STATUS_SUCCESS; return PXENV_EXIT_SUCCESS; diff --git a/src/arch/x86/interface/pxe/pxe_preboot.c b/src/arch/x86/interface/pxe/pxe_preboot.c index 09e721b34..863aaae9a 100644 --- a/src/arch/x86/interface/pxe/pxe_preboot.c +++ b/src/arch/x86/interface/pxe/pxe_preboot.c @@ -33,7 +33,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdint.h> #include <string.h> #include <stdlib.h> -#include <ipxe/uaccess.h> #include <ipxe/dhcp.h> #include <ipxe/fakedhcp.h> #include <ipxe/device.h> @@ -48,7 +47,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include "pxe_call.h" /* Avoid dragging in isapnp.o unnecessarily */ -uint16_t isapnp_read_port; +uint16_t isapnp_read_port __attribute__ (( weak )); /** Zero-based versions of PXENV_GET_CACHED_INFO::PacketType */ enum pxe_cached_info_indices { @@ -184,7 +183,7 @@ pxenv_get_cached_info ( struct s_PXENV_GET_CACHED_INFO *get_cached_info ) { union pxe_cached_info *info; unsigned int idx; size_t len; - userptr_t buffer; + void *buffer; DBGC ( &pxe_netdev, "PXENV_GET_CACHED_INFO %s to %04x:%04x+%x", pxenv_get_cached_info_name ( get_cached_info->PacketType ), @@ -243,9 +242,9 @@ pxenv_get_cached_info ( struct s_PXENV_GET_CACHED_INFO *get_cached_info ) { len = sizeof ( *info ); if ( len < sizeof ( *info ) ) DBGC ( &pxe_netdev, " buffer may be too short" ); - buffer = real_to_user ( get_cached_info->Buffer.segment, + buffer = real_to_virt ( get_cached_info->Buffer.segment, get_cached_info->Buffer.offset ); - copy_to_user ( buffer, 0, info, len ); + memcpy ( buffer, info, len ); get_cached_info->BufferSize = len; } diff --git a/src/arch/x86/interface/pxe/pxe_tftp.c b/src/arch/x86/interface/pxe/pxe_tftp.c index 3b4c6d847..aa675fd13 100644 --- a/src/arch/x86/interface/pxe/pxe_tftp.c +++ b/src/arch/x86/interface/pxe/pxe_tftp.c @@ -33,7 +33,6 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); #include <stdio.h> #include <errno.h> #include <byteswap.h> -#include <ipxe/uaccess.h> #include <ipxe/in.h> #include <ipxe/tftp.h> #include <ipxe/iobuf.h> @@ -49,7 +48,7 @@ struct pxe_tftp_connection { /** Data transfer interface */ struct interface xfer; /** Data buffer */ - userptr_t buffer; + void *buffer; /** Size of data buffer */ size_t size; /** Starting offset of data buffer */ @@ -121,9 +120,8 @@ static int pxe_tftp_xfer_deliver ( struct pxe_tftp_connection *pxe_tftp, ( pxe_tftp->start + pxe_tftp->size ) ); rc = -ENOBUFS; } else { - copy_to_user ( pxe_tftp->buffer, - ( pxe_tftp->offset - pxe_tftp->start ), - iobuf->data, len ); + memcpy ( ( pxe_tftp->buffer + pxe_tftp->offset - + pxe_tftp->start ), iobuf->data, len ); } /* Calculate new buffer position */ @@ -378,14 +376,14 @@ static PXENV_EXIT_t pxenv_tftp_read ( struct s_PXENV_TFTP_READ *tftp_read ) { tftp_read->Buffer.segment, tftp_read->Buffer.offset ); /* Read single block into buffer */ - pxe_tftp.buffer = real_to_user ( tftp_read->Buffer.segment, + pxe_tftp.buffer = real_to_virt ( tftp_read->Buffer.segment, tftp_read->Buffer.offset ); pxe_tftp.size = pxe_tftp.blksize; pxe_tftp.start = pxe_tftp.offset; while ( ( ( rc = pxe_tftp.rc ) == -EINPROGRESS ) && ( pxe_tftp.offset == pxe_tftp.start ) ) step(); - pxe_tftp.buffer = UNULL; + pxe_tftp.buffer = NULL; tftp_read->BufferSize = ( pxe_tftp.offset - pxe_tftp.start ); tftp_read->PacketNumber = ++pxe_tftp.blkidx; @@ -492,11 +490,11 @@ PXENV_EXIT_t pxenv_tftp_read_file ( struct s_PXENV_TFTP_READ_FILE } /* Read entire file */ - pxe_tftp.buffer = phys_to_user ( tftp_read_file->Buffer ); + pxe_tftp.buffer = phys_to_virt ( tftp_read_file->Buffer ); pxe_tftp.size = tftp_read_file->BufferSize; while ( ( rc = pxe_tftp.rc ) == -EINPROGRESS ) step(); - pxe_tftp.buffer = UNULL; + pxe_tftp.buffer = NULL; tftp_read_file->BufferSize = pxe_tftp.max_offset; /* Close TFTP file */ diff --git a/src/arch/x86/interface/pxe/pxe_udp.c b/src/arch/x86/interface/pxe/pxe_udp.c index a5d5eb77b..61c858dde 100644 --- a/src/arch/x86/interface/pxe/pxe_udp.c +++ b/src/arch/x86/interface/pxe/pxe_udp.c @@ -9,7 +9,6 @@ #include <ipxe/iobuf.h> #include <ipxe/xfer.h> #include <ipxe/udp.h> -#include <ipxe/uaccess.h> #include <ipxe/process.h> #include <ipxe/netdevice.h> #include <ipxe/malloc.h> @@ -296,7 +295,7 @@ pxenv_udp_write ( struct s_PXENV_UDP_WRITE *pxenv_udp_write ) { }; size_t len; struct io_buffer *iobuf; - userptr_t buffer; + const void *buffer; int rc; DBG ( "PXENV_UDP_WRITE" ); @@ -328,9 +327,9 @@ pxenv_udp_write ( struct s_PXENV_UDP_WRITE *pxenv_udp_write ) { pxenv_udp_write->Status = PXENV_STATUS_OUT_OF_RESOURCES; return PXENV_EXIT_FAILURE; } - buffer = real_to_user ( pxenv_udp_write->buffer.segment, + buffer = real_to_virt ( pxenv_udp_write->buffer.segment, pxenv_udp_write->buffer.offset ); - copy_from_user ( iob_put ( iobuf, len ), buffer, 0, len ); + memcpy ( iob_put ( iobuf, len ), buffer, len ); DBG ( " %04x:%04x+%x %d->%s:%d\n", pxenv_udp_write->buffer.segment, pxenv_udp_write->buffer.offset, pxenv_udp_write->buffer_size, @@ -400,7 +399,7 @@ static PXENV_EXIT_t pxenv_udp_read ( struct s_PXENV_UDP_READ *pxenv_udp_read ) { struct pxe_udp_pseudo_header *pshdr; uint16_t d_port_wanted = pxenv_udp_read->d_port; uint16_t d_port; - userptr_t buffer; + void *buffer; size_t len; /* Try receiving a packet, if the queue is empty */ @@ -438,12 +437,12 @@ static PXENV_EXIT_t pxenv_udp_read ( struct s_PXENV_UDP_READ *pxenv_udp_read ) { } /* Copy packet to buffer and record length */ - buffer = real_to_user ( pxenv_udp_read->buffer.segment, + buffer = real_to_virt ( pxenv_udp_read->buffer.segment, pxenv_udp_read->buffer.offset ); len = iob_len ( iobuf ); if ( len > pxenv_udp_read->buffer_size ) len = pxenv_udp_read->buffer_size; - copy_to_user ( buffer, 0, iobuf->data, len ); + memcpy ( buffer, iobuf->data, len ); pxenv_udp_read->buffer_size = len; /* Fill in source/dest information */ diff --git a/src/arch/x86/interface/syslinux/com32_call.c b/src/arch/x86/interface/syslinux/com32_call.c index 19fdbaff9..a23f46436 100644 --- a/src/arch/x86/interface/syslinux/com32_call.c +++ b/src/arch/x86/interface/syslinux/com32_call.c @@ -49,9 +49,8 @@ void __asmcall com32_intcall ( uint8_t interrupt, physaddr_t inregs_phys, physad DBGC ( &com32_regs, "COM32 INT%x in %#08lx out %#08lx\n", interrupt, inregs_phys, outregs_phys ); - memcpy_user ( virt_to_user( &com32_regs ), 0, - phys_to_user ( inregs_phys ), 0, - sizeof(com32sys_t) ); + memcpy ( &com32_regs, phys_to_virt ( inregs_phys ), + sizeof ( com32sys_t ) ); com32_int_vector = interrupt; @@ -108,9 +107,8 @@ void __asmcall com32_intcall ( uint8_t interrupt, physaddr_t inregs_phys, physad : : ); if ( outregs_phys ) { - memcpy_user ( phys_to_user ( outregs_phys ), 0, - virt_to_user( &com32_regs ), 0, - sizeof(com32sys_t) ); + memcpy ( phys_to_virt ( outregs_phys ), + &com32_regs, sizeof ( com32sys_t ) ); } } @@ -122,9 +120,8 @@ void __asmcall com32_farcall ( uint32_t proc, physaddr_t inregs_phys, physaddr_t DBGC ( &com32_regs, "COM32 farcall %04x:%04x in %#08lx out %#08lx\n", ( proc >> 16 ), ( proc & 0xffff ), inregs_phys, outregs_phys ); - memcpy_user ( virt_to_user( &com32_regs ), 0, - phys_to_user ( inregs_phys ), 0, - sizeof(com32sys_t) ); + memcpy ( &com32_regs, phys_to_virt ( inregs_phys ), + sizeof ( com32sys_t ) ); com32_farcall_proc = proc; @@ -170,9 +167,8 @@ void __asmcall com32_farcall ( uint32_t proc, physaddr_t inregs_phys, physaddr_t : : ); if ( outregs_phys ) { - memcpy_user ( phys_to_user ( outregs_phys ), 0, - virt_to_user( &com32_regs ), 0, - sizeof(com32sys_t) ); + memcpy ( phys_to_virt ( outregs_phys ), + &com32_regs, sizeof ( com32sys_t ) ); } } @@ -185,7 +181,7 @@ int __asmcall com32_cfarcall ( uint32_t proc, physaddr_t stack, size_t stacksz ) DBGC ( &com32_regs, "COM32 cfarcall %04x:%04x params %#08lx+%#zx\n", ( proc >> 16 ), ( proc & 0xffff ), stack, stacksz ); - copy_user_to_rm_stack ( phys_to_user ( stack ), stacksz ); + copy_to_rm_stack ( phys_to_virt ( stack ), stacksz ); com32_farcall_proc = proc; __asm__ __volatile__ ( @@ -194,7 +190,7 @@ int __asmcall com32_cfarcall ( uint32_t proc, physaddr_t stack, size_t stacksz ) : : "ecx", "edx" ); - remove_user_from_rm_stack ( 0, stacksz ); + remove_from_rm_stack ( NULL, stacksz ); return eax; } diff --git a/src/arch/x86/interface/syslinux/comboot_call.c b/src/arch/x86/interface/syslinux/comboot_call.c index b75e8ef7c..c3e921075 100644 --- a/src/arch/x86/interface/syslinux/comboot_call.c +++ b/src/arch/x86/interface/syslinux/comboot_call.c @@ -37,6 +37,7 @@ FILE_LICENCE ( GPL2_OR_LATER ); #include <ipxe/posix_io.h> #include <ipxe/process.h> #include <ipxe/serial.h> +#include <ipxe/ns16550.h> #include <ipxe/init.h> #include <ipxe/image.h> #include <ipxe/version.h> @@ -88,14 +89,9 @@ static uint16_t comboot_graphics_mode = 0; * Print a string with a particular terminator */ static void print_user_string ( unsigned int segment, unsigned int offset, char terminator ) { - int i = 0; - char c; - userptr_t str = real_to_user ( segment, offset ); - for ( ; ; ) { - copy_from_user ( &c, str, i, 1 ); - if ( c == terminator ) break; - putchar ( c ); - i++; + char *c; + for ( c = real_to_virt ( segment, offset ) ; *c != terminator ; c++ ) { + putchar ( *c ); } } @@ -109,26 +105,26 @@ static void shuffle ( unsigned int list_segment, unsigned int list_offset, unsig unsigned int i; /* Copy shuffle descriptor list so it doesn't get overwritten */ - copy_from_user ( shuf, real_to_user ( list_segment, list_offset ), 0, - count * sizeof( comboot_shuffle_descriptor ) ); + memcpy ( shuf, real_to_virt ( list_segment, list_offset ), + count * sizeof( comboot_shuffle_descriptor ) ); /* Do the copies */ for ( i = 0; i < count; i++ ) { - userptr_t src_u = phys_to_user ( shuf[ i ].src ); - userptr_t dest_u = phys_to_user ( shuf[ i ].dest ); + const void *src = phys_to_virt ( shuf[ i ].src ); + void *dest = phys_to_virt ( shuf[ i ].dest ); if ( shuf[ i ].src == 0xFFFFFFFF ) { /* Fill with 0 instead of copying */ - memset_user ( dest_u, 0, 0, shuf[ i ].len ); + memset ( dest, 0, shuf[ i ].len ); } else if ( shuf[ i ].dest == 0xFFFFFFFF ) { /* Copy new list of descriptors */ count = shuf[ i ].len / sizeof( comboot_shuffle_descriptor ); assert ( count <= COMBOOT_MAX_SHUFFLE_DESCRIPTORS ); - copy_from_user ( shuf, src_u, 0, shuf[ i ].len ); + memcpy ( shuf, src, shuf[ i ].len ); i = -1; } else { /* Regular copy */ - memmove_user ( dest_u, 0, src_u, 0, shuf[ i ].len ); + memmove ( dest, src, shuf[ i ].len ); } } } @@ -164,7 +160,7 @@ void comboot_force_text_mode ( void ) { /** * Fetch kernel and optional initrd */ -static int comboot_fetch_kernel ( char *kernel_file, char *cmdline ) { +static int comboot_fetch_kernel ( const char *kernel_file, char *cmdline ) { struct image *kernel; struct image *initrd; char *initrd_file; @@ -258,8 +254,8 @@ static __asmcall __used void int21 ( struct i386_all_regs *ix86 ) { break; case 0x04: /* Write Character to Serial Port */ - if ( serial_console.base ) { - uart_transmit ( &serial_console, ix86->regs.dl ); + if ( serial_console ) { + uart_transmit ( serial_console, ix86->regs.dl ); ix86->flags &= ~CF; } break; @@ -346,10 +342,8 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { case 0x0003: /* Run command */ { - userptr_t cmd_u = real_to_user ( ix86->segs.es, ix86->regs.bx ); - int len = strlen_user ( cmd_u, 0 ); - char cmd[len + 1]; - copy_from_user ( cmd, cmd_u, 0, len + 1 ); + const char *cmd = real_to_virt ( ix86->segs.es, + ix86->regs.bx ); DBG ( "COMBOOT: executing command '%s'\n", cmd ); system ( cmd ); DBG ( "COMBOOT: exiting after executing command...\n" ); @@ -370,11 +364,8 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { case 0x0006: /* Open file */ { int fd; - userptr_t file_u = real_to_user ( ix86->segs.es, ix86->regs.si ); - int len = strlen_user ( file_u, 0 ); - char file[len + 1]; - - copy_from_user ( file, file_u, 0, len + 1 ); + const char *file = real_to_virt ( ix86->segs.es, + ix86->regs.si ); if ( file[0] == '\0' ) { DBG ( "COMBOOT: attempted open with empty file name\n" ); @@ -410,7 +401,8 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { int len = ix86->regs.cx * COMBOOT_FILE_BLOCKSZ; int rc; fd_set fds; - userptr_t buf = real_to_user ( ix86->segs.es, ix86->regs.bx ); + void *buf = real_to_virt ( ix86->segs.es, + ix86->regs.bx ); /* Wait for data ready to read */ FD_ZERO ( &fds ); @@ -418,7 +410,7 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { select ( &fds, 1 ); - rc = read_user ( fd, buf, 0, len ); + rc = read ( fd, buf, len ); if ( rc < 0 ) { DBG ( "COMBOOT: read failed\n" ); ix86->regs.si = 0; @@ -454,9 +446,11 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { break; case 0x000B: /* Get Serial Console Configuration */ - if ( serial_console.base ) { - ix86->regs.dx = ( ( intptr_t ) serial_console.base ); - ix86->regs.cx = serial_console.divisor; + if ( serial_console ) { + struct ns16550_uart *comport = serial_console->priv; + + ix86->regs.dx = ( ( intptr_t ) comport->base ); + ix86->regs.cx = comport->divisor; ix86->regs.bx = 0; ix86->flags &= ~CF; } @@ -483,13 +477,10 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { case 0x0010: /* Resolve hostname */ { - userptr_t hostname_u = real_to_user ( ix86->segs.es, ix86->regs.bx ); - int len = strlen_user ( hostname_u, 0 ); - char hostname[len]; + const char *hostname = real_to_virt ( ix86->segs.es, + ix86->regs.bx ); struct in_addr addr; - copy_from_user ( hostname, hostname_u, 0, len + 1 ); - /* TODO: * "If the hostname does not contain a dot (.), the * local domain name is automatically appended." @@ -526,8 +517,8 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { "lret\n\t" ) : - : "r" ( ix86->segs.ds ), - "r" ( ix86->regs.ebp ), + : "R" ( ix86->segs.ds ), + "R" ( ix86->regs.ebp ), "d" ( ix86->regs.ebx ), "S" ( ix86->regs.esi ) ); @@ -549,15 +540,10 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { case 0x0016: /* Run kernel image */ { - userptr_t file_u = real_to_user ( ix86->segs.ds, ix86->regs.si ); - userptr_t cmd_u = real_to_user ( ix86->segs.es, ix86->regs.bx ); - int file_len = strlen_user ( file_u, 0 ); - int cmd_len = strlen_user ( cmd_u, 0 ); - char file[file_len + 1]; - char cmd[cmd_len + 1]; - - copy_from_user ( file, file_u, 0, file_len + 1 ); - copy_from_user ( cmd, cmd_u, 0, cmd_len + 1 ); + const char *file = real_to_virt ( ix86->segs.ds, + ix86->regs.si ); + char *cmd = real_to_virt ( ix86->segs.es, + ix86->regs.bx ); DBG ( "COMBOOT: run kernel %s %s\n", file, cmd ); comboot_fetch_kernel ( file, cmd ); @@ -595,9 +581,9 @@ static __asmcall __used void int22 ( struct i386_all_regs *ix86 ) { shuffle ( ix86->segs.es, ix86->regs.di, ix86->regs.cx ); /* Copy initial register values to .text16 */ - memcpy_user ( real_to_user ( rm_cs, (unsigned) __from_text16 ( &comboot_initial_regs ) ), 0, - real_to_user ( ix86->segs.ds, ix86->regs.si ), 0, - sizeof(syslinux_rm_regs) ); + memcpy ( real_to_virt ( rm_cs, (unsigned) __from_text16 ( &comboot_initial_regs ) ), + real_to_virt ( ix86->segs.ds, ix86->regs.si ), + sizeof(syslinux_rm_regs) ); /* Load initial register values */ __asm__ __volatile__ ( @@ -702,4 +688,4 @@ void unhook_comboot_interrupts ( ) { } /* Avoid dragging in serial console support unconditionally */ -struct uart serial_console __attribute__ (( weak )); +struct uart *serial_console __attribute__ (( weak )); diff --git a/src/arch/x86/interface/vmware/guestinfo.c b/src/arch/x86/interface/vmware/guestinfo.c index 4134515c1..c181d96e9 100644 --- a/src/arch/x86/interface/vmware/guestinfo.c +++ b/src/arch/x86/interface/vmware/guestinfo.c @@ -200,6 +200,7 @@ static void guestinfo_init ( void ) { /** GuestInfo settings initialiser */ struct init_fn guestinfo_init_fn __init_fn ( INIT_NORMAL ) = { + .name = "guestinfo", .initialise = guestinfo_init, }; diff --git a/src/arch/x86/interface/vmware/vmconsole.c b/src/arch/x86/interface/vmware/vmconsole.c index f7df4f75b..3b892c837 100644 --- a/src/arch/x86/interface/vmware/vmconsole.c +++ b/src/arch/x86/interface/vmware/vmconsole.c @@ -134,5 +134,6 @@ static void vmconsole_init ( void ) { * VMware logfile console initialisation function */ struct init_fn vmconsole_init_fn __init_fn ( INIT_CONSOLE ) = { + .name = "vmconsole", .initialise = vmconsole_init, }; diff --git a/src/arch/x86/prefix/lkrnprefix.S b/src/arch/x86/prefix/lkrnprefix.S index c8a04c9d7..19d30141b 100644 --- a/src/arch/x86/prefix/lkrnprefix.S +++ b/src/arch/x86/prefix/lkrnprefix.S @@ -104,6 +104,7 @@ hardware_subarch: hardware_subarch_data: .byte 0, 0, 0, 0, 0, 0, 0, 0 + .section ".prefix.data", "aw", @progbits version_string: .asciz VERSION @@ -113,6 +114,7 @@ version_string: * */ + .section ".prefix", "ax", @progbits setup: /* Fix up code segment */ pushw %ds diff --git a/src/arch/x86/prefix/pxeprefix.S b/src/arch/x86/prefix/pxeprefix.S index 5181ef618..067af99fa 100644 --- a/src/arch/x86/prefix/pxeprefix.S +++ b/src/arch/x86/prefix/pxeprefix.S @@ -1,6 +1,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ) #define PXENV_UNDI_SHUTDOWN 0x0005 +#define PXENV_UNDI_GET_INFORMATION 0x000c #define PXENV_UNDI_GET_NIC_TYPE 0x0012 #define PXENV_UNDI_GET_IFACE_INFO 0x0013 #define PXENV_STOP_UNDI 0x0015 @@ -325,18 +326,37 @@ print_structure_information: ***************************************************************************** */ get_physical_device: + /* Allow for devices that fail to set the physical device type */ + movb $0xeb, %al + movb %al, ( pxe_parameter_structure + 0x02 ) /* Issue PXENV_UNDI_GET_NIC_TYPE */ movw $PXENV_UNDI_GET_NIC_TYPE, %bx call pxe_call jnc 1f call print_pxe_error + movw $10f, %si + call print_message jmp no_physical_device 1: /* Determine physical device type */ + movw $10f, %si + call print_message movb ( pxe_parameter_structure + 0x02 ), %al cmpb $2, %al je pci_physical_device + cmpb $0xeb, %al + je probably_pci_physical_device jmp no_physical_device + .section ".prefix.data", "aw", @progbits +10: .asciz " UNDI device is " + .previous +probably_pci_physical_device: + /* Device did not write the type byte: assume PCI */ + movw $10f, %si + call print_message + .section ".prefix.data", "aw", @progbits +10: .asciz "probably " + .previous pci_physical_device: /* Record PCI bus:dev.fn and vendor/device IDs */ movl ( pxe_parameter_structure + 0x03 ), %eax @@ -346,9 +366,17 @@ pci_physical_device: movw $10f, %si call print_message call print_pci_busdevfn + movb $( ' ' ), %al + call print_character + movw pci_vendor, %ax + call print_hex_word + movb $( ':' ), %al + call print_character + movw pci_device, %ax + call print_hex_word jmp 99f .section ".prefix.data", "aw", @progbits -10: .asciz " UNDI device is PCI " +10: .asciz "PCI " .previous no_physical_device: @@ -356,12 +384,39 @@ no_physical_device: movw $10f, %si call print_message .section ".prefix.data", "aw", @progbits -10: .asciz " Unable to determine UNDI physical device" +10: .asciz "unknown" .previous 99: /***************************************************************************** + * Get IRQ number + ***************************************************************************** + */ +get_irq: + /* Issue PXENV_UNDI_GET_INFORMATION */ + movw $PXENV_UNDI_GET_INFORMATION, %bx + call pxe_call + jnc 1f + call print_pxe_error + jmp 99f +1: /* Check for a valid IRQ number */ + movw ( pxe_parameter_structure + 0x04 ), %ax + testw %ax, %ax + jz 99f + cmpw $15, %ax + ja 99f + /* Store IRQ number */ + movw %ax, undi_irq + movw $10f, %si + call print_message + call print_word + .section ".prefix.data", "aw", @progbits +10: .asciz ", IRQ " + .previous +99: + +/***************************************************************************** * Determine interface type ***************************************************************************** */ @@ -452,6 +507,30 @@ pxe_cmdline: .previous /***************************************************************************** + * Ensure NIC interrupt is disabled + ***************************************************************************** + */ +disable_irq: + /* Check for a recorded IRQ number */ + movw undi_irq, %cx + testw %cx, %cx + jz 99f + /* Calculate IMR */ + movw %cx, %dx + shlw $4, %dx + andb $0x80, %dl + orb $0x21, %dl + /* Calculate mask value */ + movb $0x01, %bl + andb $0x07, %cl + shlb %cl, %bl + /* Mask interrupt */ + inb %dx, %al + orb %bl, %al + outb %al, %dx +99: + +/***************************************************************************** * Leave NIC in a safe state ***************************************************************************** */ @@ -740,6 +819,8 @@ undi_data_segoff: undi_data_size: .word 0 undi_data_segment: .word 0 +undi_irq: .word 0 + pxe_hacks: .word 0 /* The following fields are part of a struct undi_device */ diff --git a/src/arch/x86/prefix/unlzma.S b/src/arch/x86/prefix/unlzma.S index e4d1e190d..6ab3222e2 100644 --- a/src/arch/x86/prefix/unlzma.S +++ b/src/arch/x86/prefix/unlzma.S @@ -45,7 +45,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); .section ".note.GNU-stack", "", @progbits .code32 - .arch i486 + .arch i386 .section ".prefix.lib", "ax", @progbits #ifdef CODE16 @@ -962,7 +962,9 @@ decompress: ADDR32 lodsb /* discard initial byte */ print_hex_byte %al ADDR32 lodsl - bswapl %eax + xchgb %al, %ah + roll $16, %eax + xchgb %al, %ah print_hex_dword %eax print_character $('\n') movl %eax, rc_code(%ebp) diff --git a/src/arch/x86/scripts/pcbios.lds b/src/arch/x86/scripts/pcbios.lds index e208b174b..f8c137283 100644 --- a/src/arch/x86/scripts/pcbios.lds +++ b/src/arch/x86/scripts/pcbios.lds @@ -130,8 +130,12 @@ SECTIONS { . += 1; /* Prevent NULL being valid */ *(.text) *(.text.*) + *(.srodata) + *(.srodata.*) *(.rodata) *(.rodata.*) + *(.sdata) + *(.sdata.*) *(.data) *(.data.*) KEEP(*(SORT(.tbl.*))) /* Various tables. See include/tables.h */ @@ -139,6 +143,8 @@ SECTIONS { KEEP(*(.provided.*)) _mtextdata = .; } .bss.textdata (NOLOAD) : AT ( _bss_textdata_lma ) { + *(.sbss) + *(.sbss.*) *(.bss) *(.bss.*) *(COMMON) diff --git a/src/arch/x86/transitions/librm_mgmt.c b/src/arch/x86/transitions/librm_mgmt.c index b3820589c..89feec96a 100644 --- a/src/arch/x86/transitions/librm_mgmt.c +++ b/src/arch/x86/transitions/librm_mgmt.c @@ -58,35 +58,36 @@ static struct profiler timer_irq_profiler __profiler = { .name = "irq.timer" }; static struct profiler other_irq_profiler __profiler = { .name = "irq.other" }; /** - * Allocate space on the real-mode stack and copy data there from a - * user buffer + * Allocate space on the real-mode stack and copy data there * - * @v data User buffer + * @v data Stack data * @v size Size of stack data * @ret sp New value of real-mode stack pointer */ -uint16_t copy_user_to_rm_stack ( userptr_t data, size_t size ) { - userptr_t rm_stack; +uint16_t copy_to_rm_stack ( const void *data, size_t size ) { + void *rm_stack; + rm_sp -= size; - rm_stack = real_to_user ( rm_ss, rm_sp ); - memcpy_user ( rm_stack, 0, data, 0, size ); + rm_stack = real_to_virt ( rm_ss, rm_sp ); + memcpy ( rm_stack, data, size ); return rm_sp; -}; +} /** - * Deallocate space on the real-mode stack, optionally copying back - * data to a user buffer. + * Deallocate space on the real-mode stack, optionally copying back data * - * @v data User buffer + * @v data Stack data buffer, or NULL * @v size Size of stack data */ -void remove_user_from_rm_stack ( userptr_t data, size_t size ) { +void remove_from_rm_stack ( void *data, size_t size ) { + const void *rm_stack; + if ( data ) { - userptr_t rm_stack = real_to_user ( rm_ss, rm_sp ); - memcpy_user ( rm_stack, 0, data, 0, size ); + rm_stack = real_to_virt ( rm_ss, rm_sp ); + memcpy ( data, rm_stack, size ); } rm_sp += size; -}; +} /** * Set interrupt vector @@ -302,17 +303,14 @@ static void * ioremap_pages ( unsigned long bus_addr, size_t len ) { /* Calculate number of pages required */ count = ( ( offset + len + IO_PAGE_SIZE - 1 ) / IO_PAGE_SIZE ); assert ( count != 0 ); - assert ( count < ( sizeof ( io_pages.page ) / - sizeof ( io_pages.page[0] ) ) ); + assert ( count <= IO_PAGE_COUNT ); /* Round up number of pages to a power of two */ - stride = ( 1 << ( fls ( count ) - 1 ) ); + stride = ( 1 << fls ( count - 1 ) ); assert ( count <= stride ); /* Allocate pages */ - for ( first = 0 ; first < ( sizeof ( io_pages.page ) / - sizeof ( io_pages.page[0] ) ) ; - first += stride ) { + for ( first = 0 ; first < IO_PAGE_COUNT ; first += stride ) { /* Calculate I/O address */ io_addr = ( IO_BASE + ( first * IO_PAGE_SIZE ) + offset ); @@ -365,6 +363,10 @@ static void iounmap_pages ( volatile const void *io_addr ) { /* Calculate first page table entry */ first = ( ( io_addr - IO_BASE ) / IO_PAGE_SIZE ); + /* Ignore unmappings outside of the I/O range */ + if ( first >= IO_PAGE_COUNT ) + return; + /* Clear page table entries */ for ( i = first ; ; i++ ) { @@ -425,19 +427,9 @@ void setup_sipi ( unsigned int vector, uint32_t handler, memcpy ( &sipi_regs, regs, sizeof ( sipi_regs ) ); /* Copy real-mode handler */ - copy_to_real ( ( vector << 8 ), 0, sipi, ( ( size_t ) sipi_len ) ); + copy_to_real ( ( vector << 8 ), 0, sipi, sipi_len ); } -PROVIDE_UACCESS_INLINE ( librm, phys_to_user ); -PROVIDE_UACCESS_INLINE ( librm, user_to_phys ); -PROVIDE_UACCESS_INLINE ( librm, virt_to_user ); -PROVIDE_UACCESS_INLINE ( librm, user_to_virt ); -PROVIDE_UACCESS_INLINE ( librm, userptr_add ); -PROVIDE_UACCESS_INLINE ( librm, memcpy_user ); -PROVIDE_UACCESS_INLINE ( librm, memmove_user ); -PROVIDE_UACCESS_INLINE ( librm, memset_user ); -PROVIDE_UACCESS_INLINE ( librm, strlen_user ); -PROVIDE_UACCESS_INLINE ( librm, memchr_user ); PROVIDE_IOMAP_INLINE ( pages, io_to_bus ); PROVIDE_IOMAP ( pages, ioremap, ioremap_pages ); PROVIDE_IOMAP ( pages, iounmap, iounmap_pages ); diff --git a/src/arch/x86_64/Makefile b/src/arch/x86_64/Makefile index b3064b752..276b96d3b 100644 --- a/src/arch/x86_64/Makefile +++ b/src/arch/x86_64/Makefile @@ -1,3 +1,7 @@ +# Specify compressor +# +ZBIN = $(ZBIN64) + # Code size reduction. # CFLAGS += -fstrength-reduce -fomit-frame-pointer diff --git a/src/arch/x86_64/Makefile.efi b/src/arch/x86_64/Makefile.efi index 0041bb8f0..f38375607 100644 --- a/src/arch/x86_64/Makefile.efi +++ b/src/arch/x86_64/Makefile.efi @@ -12,10 +12,6 @@ CFLAGS += -mno-red-zone # ELF2EFI = $(ELF2EFI64) -# Specify EFI boot file -# -EFI_BOOT_FILE = bootx64.efi - # Include generic EFI Makefile # MAKEDEPS += arch/x86/Makefile.efi diff --git a/src/arch/x86_64/include/bits/byteswap.h b/src/arch/x86_64/include/bits/byteswap.h index d8c5098ef..7c48a27ca 100644 --- a/src/arch/x86_64/include/bits/byteswap.h +++ b/src/arch/x86_64/include/bits/byteswap.h @@ -10,6 +10,7 @@ #include <stdint.h> FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); static inline __attribute__ (( always_inline, const )) uint16_t __bswap_variable_16 ( uint16_t x ) { diff --git a/src/arch/x86_64/include/bits/compiler.h b/src/arch/x86_64/include/bits/compiler.h index 1c04a7b30..99185b058 100644 --- a/src/arch/x86_64/include/bits/compiler.h +++ b/src/arch/x86_64/include/bits/compiler.h @@ -2,6 +2,7 @@ #define _BITS_COMPILER_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /** Dummy relocation type */ #define RELOC_TYPE_NONE R_X86_64_NONE diff --git a/src/arch/x86_64/include/gdbmach.h b/src/arch/x86_64/include/bits/gdbmach.h index 367405fd6..367405fd6 100644 --- a/src/arch/x86_64/include/gdbmach.h +++ b/src/arch/x86_64/include/bits/gdbmach.h diff --git a/src/arch/x86_64/include/bits/profile.h b/src/arch/x86_64/include/bits/profile.h index b7c74fbe7..c8e0a21f1 100644 --- a/src/arch/x86_64/include/bits/profile.h +++ b/src/arch/x86_64/include/bits/profile.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <stdint.h> @@ -16,7 +17,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); * * @ret timestamp Timestamp */ -static inline __attribute__ (( always_inline )) uint64_t +static inline __attribute__ (( always_inline )) unsigned long profile_timestamp ( void ) { uint32_t eax; uint32_t edx; diff --git a/src/arch/x86_64/include/setjmp.h b/src/arch/x86_64/include/bits/setjmp.h index 69835d9fa..adfb869ea 100644 --- a/src/arch/x86_64/include/setjmp.h +++ b/src/arch/x86_64/include/bits/setjmp.h @@ -1,5 +1,5 @@ -#ifndef _SETJMP_H -#define _SETJMP_H +#ifndef _BITS_SETJMP_H +#define _BITS_SETJMP_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); @@ -25,10 +25,4 @@ typedef struct { uint64_t r15; } jmp_buf[1]; -extern int __asmcall __attribute__ (( returns_twice )) -setjmp ( jmp_buf env ); - -extern void __asmcall __attribute__ (( noreturn )) -longjmp ( jmp_buf env, int val ); - -#endif /* _SETJMP_H */ +#endif /* _BITS_SETJMP_H */ diff --git a/src/arch/x86_64/include/bits/stdint.h b/src/arch/x86_64/include/bits/stdint.h index fe1f9946a..e75bed502 100644 --- a/src/arch/x86_64/include/bits/stdint.h +++ b/src/arch/x86_64/include/bits/stdint.h @@ -2,6 +2,7 @@ #define _BITS_STDINT_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); typedef __SIZE_TYPE__ size_t; typedef signed long ssize_t; diff --git a/src/arch/x86_64/include/bits/strings.h b/src/arch/x86_64/include/bits/strings.h index 3b7911f3b..6da8f1350 100644 --- a/src/arch/x86_64/include/bits/strings.h +++ b/src/arch/x86_64/include/bits/strings.h @@ -2,6 +2,7 @@ #define _BITS_STRINGS_H FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /** * Find first (i.e. least significant) set bit diff --git a/src/arch/x86_64/include/ipxe/efi/dhcparch.h b/src/arch/x86_64/include/ipxe/efi/dhcparch.h index ccf0f46a0..f75bf9145 100644 --- a/src/arch/x86_64/include/ipxe/efi/dhcparch.h +++ b/src/arch/x86_64/include/ipxe/efi/dhcparch.h @@ -8,6 +8,7 @@ */ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); #include <ipxe/dhcp.h> diff --git a/src/arch/x86_64/include/limits.h b/src/arch/x86_64/include/limits.h index a1374a17f..e75461acb 100644 --- a/src/arch/x86_64/include/limits.h +++ b/src/arch/x86_64/include/limits.h @@ -2,6 +2,7 @@ #define LIMITS_H 1 FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL ); +FILE_SECBOOT ( PERMITTED ); /* Number of bits in a `char' */ #define CHAR_BIT 8 |