path: root/src/arch/riscv/prefix/libprefix.S

/*
 * Copyright (C) 2025 Michael Brown <mbrown@fensystems.co.uk>.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
 * 02110-1301, USA.
 *
 * You can also choose to distribute this program under the terms of
 * the Unmodified Binary Distribution Licence (as given in the file
 * COPYING.UBDL), provided that you have satisfied its requirements.
 */

	FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL )

/** @file
 *
 * RISC-V prefix library
 *
 */

#include <config/serial.h>
#include <config/fault.h>

	.section ".note.GNU-stack", "", @progbits
	.text

	/* Link-time base address of _prefix
	 *
	 * This will be not be updated if runtime relocations are applied.
	 */
	.section ".rodata.prefix_link", "a", @progbits
	.balign	( __riscv_xlen / 8 )
prefix_link:
	.dword	_base
	.size	prefix_link, . - prefix_link

	/* Virtual address of _prefix
	 *
	 * This will be updated if runtime relocations are applied.
	 */
	.section ".rodata.prefix_virt", "a", @progbits
	.balign	( __riscv_xlen / 8 )
prefix_virt:
	.dword	_prefix
	.size	prefix_virt, . - prefix_virt

/*****************************************************************************
 *
 * Print character via debug console extension
 *
 *****************************************************************************
 *
 * Print a single character via the SBI DBCN extension.
 *
 * Parameters:
 *
 *   a0 - Character to print
 *
 * Returns:
 *
 *   a0 - Zero if character printed successfully
 *   a1 - Overwritten
 *   a6 - Overwritten
 *   a7 - Overwritten
 *
 */

/* SBI debug console extension */
#define SBI_DBCN ( ( 'D' << 24 ) | ( 'B' << 16 ) | ( 'C' << 8 ) | 'N' )
#define SBI_DBCN_WRITE_BYTE 0x02

	.macro	print_char_dbcn
	li	a7, SBI_DBCN
	li	a6, SBI_DBCN_WRITE_BYTE
	ecall
	.endm

/*****************************************************************************
 *
 * Print character via legacy extension
 *
 *****************************************************************************
 *
 * Print a single character via the SBI putchar legacy extension.
 *
 * Parameters:
 *
 *   a0 - Character to print
 *
 * Returns:
 *
 *   a0 - Overwritten
 *   a7 - Overwritten
 *
 */

/* SBI legacy console putchar */
#define SBI_LEGACY_PUTCHAR 0x01

	.macro	print_char_legacy
	li	a7, SBI_LEGACY_PUTCHAR
	ecall
	.endm

/*****************************************************************************
 *
 * Print character via early UART
 *
 *****************************************************************************
 *
 * Print a single character via a UART.
 *
 * For devices without a functional SBI console, a UART at a hardcoded
 * address can be used as a last resort mechanism for obtaining debug
 * output from the prefix.
 *
 * Parameters:
 *
 *   a0 - Character to print
 *
 * Returns:
 *
 *   a0 - Preserved
 *   a1 - May be overwritten
 *   a6 - May be overwritten
 *   a7 - May be overwritten
 *
 */

/* Default to no UART, if not specified */
#ifndef EARLY_UART_MODEL
#define EARLY_UART_MODEL none
#endif

/* Default to a register shift of zero, if not specified */
#ifndef EARLY_UART_REG_SHIFT
#define EARLY_UART_REG_SHIFT 0
#endif

#define print_char_uart _C2 ( print_char_uart_, EARLY_UART_MODEL )

#define early_uart_reg_base _C2 ( early_uart_reg_base_, __riscv_xlen )

	/* Print character via nonexistent UART */
	.macro	print_char_uart_none
	.endm

	/*
	 * Get UART base address (64-bit addressing)
	 */
	.macro	early_uart_reg_base_64 reg
	csrr	\reg, satp
	beqz	\reg, early_uart_reg_base_64_nonpaged_\@
	LOADN	\reg, early_uart_reg_base_64_virt
	j	early_uart_reg_base_64_done_\@
early_uart_reg_base_64_nonpaged_\@:
	li	\reg, EARLY_UART_REG_BASE
early_uart_reg_base_64_done_\@:
	.endm

	/*
	 * Get UART base address (32-bit addressing)
	 */
	.macro	early_uart_reg_base_32 reg
	li	\reg, EARLY_UART_REG_BASE
	sub	\reg, \reg, tp
	.endm

/*****************************************************************************
 *
 * Print character via 8250-compatible early UART
 *
 *****************************************************************************
 *
 * Print a single character via an 8250- or 16550-compatible UART.
 *
 * Parameters:
 *
 *   a0 - Character to print
 *
 * Returns:
 *
 *   a0 - Preserved
 *   a1 - Overwritten
 *   a7 - Overwritten
 *
 */

/* 8250-compatible UART transmit registers */
#define EARLY_UART_8250_TX		( 0 << EARLY_UART_REG_SHIFT )
#define EARLY_UART_8250_LSR		( 5 << EARLY_UART_REG_SHIFT )
#define EARLY_UART_8250_LSR_THRE	0x20

	.macro	print_char_uart_8250
	early_uart_reg_base a7
	sb	a0, EARLY_UART_8250_TX(a7)
	fence
early_uart_8250_wait_\@:
	lbu	a1, EARLY_UART_8250_LSR(a7)
	andi	a1, a1, EARLY_UART_8250_LSR_THRE
	beqz	a1, early_uart_8250_wait_\@
	.endm

/*****************************************************************************
 *
 * Print character via SiFive-compatible early UART
 *
 *****************************************************************************
 *
 * Print a single character via a SiFive-compatible UART.
 *
 * Parameters:
 *
 *   a0 - Character to print
 *
 * Returns:
 *
 *   a0 - Preserved
 *   a1 - Overwritten
 *   a7 - Overwritten
 *
 */

/* SiFive-compatible UART transmit registers */
#define EARLY_UART_SIFIVE_TXFIFO	( 0 << EARLY_UART_REG_SHIFT )

	.macro	print_char_uart_sifive
	early_uart_reg_base a7
	sw	a0, EARLY_UART_SIFIVE_TXFIFO(a7)
	fence
early_uart_sifive_wait_\@:
	lw	a1, EARLY_UART_SIFIVE_TXFIFO(a7)
	bltz	a1, early_uart_sifive_wait_\@
	.endm

/*****************************************************************************
 *
 * Print single character to early UART (from C code)
 *
 *****************************************************************************
 *
 * This function is called by the SBI console driver to output a
 * character to the early UART (if enabled).
 *
 * The standard C ABI applies to this function.
 *
 * Parameters:
 *
 *   a0 - Character to print
 *
 * Returns: none
 *
 */

	.section ".prefix.early_uart_putchar", "ax", @progbits
	.globl	early_uart_putchar
early_uart_putchar:
	print_char_uart
	ret
	.size	early_uart_putchar, . - early_uart_putchar

/*****************************************************************************
 *
 * Print message to debug console
 *
 *****************************************************************************
 *
 * Print a NUL-terminated string to the debug console.
 *
 * This function prints one character at a time via the "write byte"
 * call (rather than using "write string"), since this avoids any need
 * to know the current virtual-physical address translation.  It does
 * not require a valid stack.
 *
 * Note that the parameter is passed in register t1 (rather than a0)
 * and all non-temporary registers are preserved.
 *
 * Parameters:
 *
 *   t1 - Pointer to string
 *
 * Returns: none
 *
 */

	.section ".prefix.print_message", "ax", @progbits
	.globl	print_message
print_message:
	/* Handle alternate link register */
	mv	t0, ra
print_message_alt:
	/* Register usage:
	 *
	 * a0 - current character
	 * t0 - alternate link register
	 * t1 - character pointer
	 * t2 - preserved a0
	 * t3 - preserved a1
	 * t4 - preserved a6
	 * t5 - preserved a7
	 */
	mv	t2, a0
	mv	t3, a1
	mv	t4, a6
	mv	t5, a7

1:	/* Print each character in turn */
	lbu	a0, (t1)
	addi	t1, t1, 1
	beqz	a0, 2f
	print_char_uart
	print_char_dbcn
	beqz	a0, 1b
	lbu	a0, -1(t1)
	print_char_legacy
	j	1b
2:
	/* Restore registers and return (via alternate link register) */
	mv	a7, t5
	mv	a6, t4
	mv	a1, t3
	mv	a0, t2
	jr	t0
	.size	print_message, . - print_message

	/*
	 * Display progress message (if debugging is enabled)
	 */
	.macro	progress message
#ifndef NDEBUG
	.section ".rodata.progress_\@", "a", @progbits
progress_\@:
	.asciz	"\message"
	.size	progress_\@, . - progress_\@
	.previous
	la	t1, progress_\@
	jal	t0, print_message_alt
#endif
	.endm

/*****************************************************************************
 *
 * Print hexadecimal value to debug console
 *
 *****************************************************************************
 *
 * Print a register value in hexadecimal to the debug console.
 *
 * This function does not require a valid stack.
 *
 * Note that the parameters are passed in registers t1 and t2 (rather
 * than a0) and all non-temporary registers are preserved.
 *
 * Parameters:
 *
 *   t1 - Value to print
 *   t2 - Number of bits to print (must be a multiple of 4)
 *
 * Returns: none
 *
 */

	/*
	 * Convert a single nibble to an ASCII character
	 */
	.macro	nibble_to_ascii reg
	addi	\reg, \reg, -10
	bltz	\reg, dec_\@
	addi	\reg, \reg, ( 'a' - ( '0' + 10 ) )
dec_\@:	addi	\reg, \reg, ( '0' + 10 )
	.endm

	.section ".prefix.print_hex_value", "ax", @progbits
	.globl	print_hex_value
print_hex_value:
	/* Handle alternate link register */
	mv	t0, ra
print_hex_value_alt:
	/* Register usage:
	 *
	 * a0 - current digit / general temporary
	 * t0 - alternate link register
	 * t1 - current value
	 * t2 - digit counter
	 * t3 - preserved a0
	 * t4 - preserved a1
	 * t5 - preserved a6
	 * t6 - preserved a7
	 */
	mv	t3, a0
	mv	t4, a1
	mv	t5, a6
	mv	t6, a7

	/* Skip any unprinted digits */
	li	a0, __riscv_xlen
	sub	a0, a0, t2
	sll	t1, t1, a0

1:	/* Print each digit in turn */
	srli	a0, t1, ( __riscv_xlen - 4 )
	nibble_to_ascii a0
	print_char_uart
	print_char_dbcn
	beqz	a0, 2f
	srli	a0, t1, ( __riscv_xlen - 4 )
	nibble_to_ascii a0
	print_char_legacy
2:	slli	t1, t1, 4
	addi	t2, t2, -4
	bgtz	t2, 1b

	/* Restore registers and return (via alternate link register) */
	mv	a7, t6
	mv	a6, t5
	mv	a1, t4
	mv	a0, t3
	jr	t0
	.size	print_hex_value, . - print_hex_value

	/*
	 * Display hexadecimal register value (if debugging is enabled)
	 */
	.macro	print_hex_reg reg, bits=__riscv_xlen
#ifndef NDEBUG
	mv	t1, \reg
	li	t2, \bits
	jal	t0, print_hex_value_alt
#endif
	.endm

	/*
	 * Display hexadecimal symbol address (if debugging is enabled)
	 */
	.macro	print_hex_addr sym
#ifndef NDEBUG
	la	t1, \sym
	li	t2, __riscv_xlen
	jal	t0, print_hex_value_alt
#endif
	.endm

	/*
	 * Display hexadecimal data value (if debugging is enabled)
	 */
	.macro	print_hex_data sym
#ifndef NDEBUG
	LOADN	t1, \sym
	li	t2, __riscv_xlen
	jal	t0, print_hex_value_alt
#endif
	.endm

/*****************************************************************************
 *
 * Apply compressed relocation records
 *
 *****************************************************************************
 *
 * Apply compressed relocation records to fix up iPXE to run at its
 * current virtual address.
 *
 * This function must run before .bss is zeroed (since the relocation
 * records are overlaid with .bss).  It does not require a valid stack
 * pointer.
 *
 * Parameters: none
 *
 *   a0 - Relocation records
 *
 * Returns: none
 *
 */

/** Number of bits in a skip value */
#define ZREL_SKIP_BITS 19

	.section ".prefix.apply_relocs", "ax", @progbits
	.globl	apply_relocs
apply_relocs:
	/* Register usage:
	 *
	 * a0 - current relocation record pointer
	 * a1 - current relocation target address
	 * a2 - relocation addend
	 * a3 - current relocation record value
	 * a4 - number of bits remaining in current relocation record
	 */
	la	a1, _prefix

	/* Calculate relocation addend */
	LOADN	a2, prefix_virt
	sub	a2, a1, a2

	/* Skip applying relocations if addend is zero */
	beqz	a2, apply_relocs_done
	progress " reloc"

	/* Test writability
	 *
	 * We do this to avoid accidentally sending an undefined
	 * sequence of commands to a flash device, if we are started
	 * from read-only memory with no paging support.
	 *
	 * We attempt to write an all-ones pattern, on the basis that
	 * this pattern will harmlessly cause any flash device
	 * conforming to the CFI01 specification to enter the default
	 * "read array" state.
	 */
	la	t0, apply_relocs_test
	li	t1, -1
	STOREN	t1, (t0)
	LOADN	t2, (t0)
	bne	t1, t2, apply_relocs_failed

apply_relocs_loop:
	/* Read new relocation record */
	LOADN	a3, (a0)
	addi	a0, a0, ( __riscv_xlen / 8 )
	li	a4, ( __riscv_xlen - 1 )

	/* Consume and apply skip, if present (i.e. if MSB=0) */
	bltz	a3, 1f
	addi	a4, a4, -ZREL_SKIP_BITS
	srli	t0, a3, ( __riscv_xlen - ( ZREL_SKIP_BITS + 1 ) )
	slli	t0, t0, ( ( __riscv_xlen / 32 ) + 1 )
	add	a1, a1, t0
1:
	/* Apply relocations corresponding to set bits in record */
1:	andi	t0, a3, 1
	beqz	t0, 2f
	LOADN	t1, (a1)
	add	t1, t1, a2
	STOREN	t1, (a1)
2:	addi	a1, a1, ( __riscv_xlen / 8 )
	srli	a3, a3, 1
	addi	a4, a4, -1
	bnez	a4, 1b

	/* Loop until we have reached a terminator record (MSB=0, offset=0) */
	bnez	a3, apply_relocs_loop

	/* Check that relocations were applied successfully */
	la	t0, _prefix
	LOADN	t1, prefix_virt
	bne	t0, t1, apply_relocs_failed

apply_relocs_done:
	/* Return to caller */
	progress " ok\r\n"
	ret

apply_relocs_failed:
	/* Failure to apply relocations (if relocations were needed)
	 * is a fatal error.
	 */
	progress " failed\r\n"
	j	reset_system
	.size	apply_relocs, . - apply_relocs

	/* Writability test
	 *
	 * Placed within .data rather than .bss, since we need this to
	 * be within the range of the stored iPXE image.
	 */
	.section ".data.apply_relocs_test", "aw", @progbits
	.balign	( __riscv_xlen / 8 )
apply_relocs_test:
	.space	( __riscv_xlen / 8 )
	.size	apply_relocs_test, . - apply_relocs_test

/*****************************************************************************
 *
 * Enable paging
 *
 *****************************************************************************
 *
 * This function must be called with flat physical addressing.  It
 * does not require a valid stack pointer.
 *
 * Parameters:
 *
 *   a0 - Page table to fill in (4kB, must be aligned to a 4kB boundary)
 *
 * Returns:
 *
 *   a0 - Size of accessible physical address space (or zero for no limit)
 *   tp - Virtual address offset
 *   pc - Updated to a virtual address if paging enabled
 *
 */

/** Number of bits in a page offset */
#define PAGE_SHIFT 12

/** Page size */
#define PAGE_SIZE ( 1 << PAGE_SHIFT )

/** Size of a page table entry (log2) */
#define PTE_SIZE_LOG2 ( ( __riscv_xlen / 32 ) + 1 )

/** Size of a page table entry */
#define PTE_SIZE ( 1 << PTE_SIZE_LOG2 )

/** Number of page table entries (log2) */
#define PTE_COUNT_LOG2 ( PAGE_SHIFT - PTE_SIZE_LOG2 )

/** Number of page table entries */
#define PTE_COUNT ( 1 << PTE_COUNT_LOG2 )

/** Number of bits in a virtual or physical page number */
#define VPPN_SHIFT PTE_COUNT_LOG2

/* Page table entry flags */
#define PTE_V		0x00000001	/**< Page table entry is valid */
#define PTE_R		0x00000002	/**< Page is readable */
#define PTE_W		0x00000004	/**< Page is writable */
#define PTE_X		0x00000008	/**< Page is executable */
#define PTE_A		0x00000040	/**< Page has been accessed */
#define PTE_D		0x00000080	/**< Page is dirty */

/* Page table entry flags for our leaf pages */
#define PTE_LEAF ( PTE_D | PTE_A | PTE_X | PTE_W | PTE_R | PTE_V )

/** Physical page number LSB in PTE */
#define PTE_PPN_LSB(x) ( 10 + (x) * VPPN_SHIFT )
#define PTE_PPN4_LSB	PTE_PPN_LSB(4)	/**< PPN[4] LSB (Sv57) */
#define PTE_PPN3_LSB	PTE_PPN_LSB(3)	/**< PPN[3] LSB (Sv57 & Sv48) */
#define PTE_PPN2_LSB	PTE_PPN_LSB(2)	/**< PPN[2] LSB (Sv57, Sv48, & Sv39) */
#define PTE_PPN1_LSB	PTE_PPN_LSB(1)	/**< PPN[1] LSB (all levels) */
#define PTE_PPN0_LSB	PTE_PPN_LSB(0)	/**< PPN[0] LSB (all levels) */

/** Page table entry physical page address shift */
#define PTE_PPN_SHIFT ( PAGE_SHIFT - PTE_PPN0_LSB )

/** Virtual page number LSB */
#define VPN_LSB(x) ( PAGE_SHIFT + (x) * VPPN_SHIFT )
#define VPN4_LSB	VPN_LSB(4)	/**< VPN[4] LSB (Sv57) */
#define VPN3_LSB	VPN_LSB(3)	/**< VPN[3] LSB (Sv57 & Sv48) */
#define VPN2_LSB	VPN_LSB(2)	/**< VPN[2] LSB (Sv57, Sv48, & Sv39) */
#define VPN1_LSB	VPN_LSB(1)	/**< VPN[1] LSB (all levels) */
#define VPN0_LSB	VPN_LSB(0)	/**< VPN[0] LSB (all levels) */

/* Paging modes */
#define SATP_MODE_SV57	10		/**< Five-level paging (Sv57) */
#define SATP_MODE_SV48	9		/**< Four-level paging (Sv48) */
#define SATP_MODE_SV39	8		/**< Three-level paging (Sv39) */
#define SATP_MODE_SV32	1		/**< Two-level paging (Sv32) */

/** Paging mode shift */
#if __riscv_xlen == 64
#define SATP_MODE_SHIFT	60
#else
#define SATP_MODE_SHIFT	31
#endif

	.globl	enable_paging
	.equ	enable_paging, _C2 ( enable_paging_, __riscv_xlen )

	/* Paging mode names (for debug messages) */
	.section ".rodata.paging_mode_names", "a", @progbits
paging_mode_names:
	.asciz	"none"
	.org	( paging_mode_names + 5 * SATP_MODE_SV32 )
	.asciz	"Sv32"
	.org	( paging_mode_names + 5 * SATP_MODE_SV39 )
	.asciz	"Sv39"
	.org	( paging_mode_names + 5 * SATP_MODE_SV48 )
	.asciz	"Sv48"
	.org	( paging_mode_names + 5 * SATP_MODE_SV57 )
	.asciz	"Sv57"
	.size	paging_mode_names, . - paging_mode_names

	/*
	 * Display paging mode name (if debugging is enabled)
	 */
	.macro	paging_mode_name reg
#ifndef NDEBUG
	slli	t0, \reg, 2
	add	t0, t0, \reg
	la	t1, paging_mode_names
	add	t1, t1, t0
	jal	t0, print_message_alt
#endif
	.endm

	/* Maximum physical alignment
	 *
	 * We align to a "megapage" boundary to simplify the task of
	 * setting up page table mappings.
	 */
	.globl	_max_align
	.equ	_max_align, ( 1 << VPN1_LSB )

	/* Space for page table
	 *
	 * This can be used only once .bss is known to be writable.
	 */
	.section ".bss.page_table", "a", @nobits
	.globl	page_table
	.balign	PAGE_SIZE
page_table:
	.space	PAGE_SIZE
	.size	page_table, . - page_table

	/* Convert physical address to virtual address */
	.macro	phys_to_virt rd, rs:vararg
	_C2 ( phys_to_virt_, __riscv_xlen ) \rd, \rs
	.endm

/*****************************************************************************
 *
 * Disable paging
 *
 *****************************************************************************
 *
 * This function may be called with either virtual or flat physical
 * addressing.  It does not require a valid stack pointer.
 *
 * Parameters:
 *
 *   tp - Virtual address offset
 *
 * Returns:
 *
 *   tp - Virtual address offset (zeroed)
 *   pc - Updated to a physical address
 *
 */

	.globl	disable_paging
	.equ	disable_paging, _C2 ( disable_paging_, __riscv_xlen )

/*****************************************************************************
 *
 * Enable 64-bit paging
 *
 *****************************************************************************
 *
 * Construct a 64-bit page table to identity-map the whole of the
 * mappable physical address space, and to map iPXE itself at its
 * link-time address (which must be 2MB-aligned and be within the
 * upper half of the kernel address space).
 *
 * This function must be called with flat physical addressing.  It
 * does not require a valid stack pointer.
 *
 * Parameters:
 *
 *   a0 - Page table to fill in (4kB, must be aligned to a 4kB boundary)
 *
 * Returns:
 *
 *   a0 - Size of accessible physical address space (or zero for no limit)
 *   tp - Virtual address offset
 *   pc - Updated to a virtual address if paging enabled
 *
 * A 4kB 64-bit page table contains 512 8-byte PTEs.  We choose to use
 * these as:
 *
 *    - PTE[0-255] : Identity map for the physical address space.
 *
 *      This conveniently requires exactly 256 PTEs, regardless of the
 *      paging level.  Higher paging levels are able to identity-map a
 *      larger physical address space:
 *
 *      Sv57 : 256 x 256TB "petapages" (55-bit physical address space)
 *      Sv48 : 256 x 512GB "terapages" (46-bit physical address space)
 *      Sv39 : 256 x   1GB "gigapages" (37-bit physical address space)
 *
 *      Note that Sv48 and Sv39 cannot identity-map the whole of the
 *      available physical address space, since the virtual address
 *      space is not large enough (and is halved by the constraint
 *      that virtual addresses with bit 47/38 set must also have all
 *      higher bits set, and so cannot identity-map to a 55-bit
 *      physical address).
 *
 *    - PTE[x-y] : Virtual address map for iPXE
 *
 *      These are 2MB "megapages" used to map the link-time virtual
 *      address range used by iPXE itself.  We can use any 2MB-aligned
 *      range within 0xffffffffe0800000-0xffffffffffc00000, which
 *      breaks down as:
 *
 *         VPN[4] = 511     (in Sv57, must be all-ones in Sv48 and Sv39)
 *         VPN[3] = 511     (in Sv57 and Sv48, must be all-ones in Sv39)
 *         VPN[2] = 511     (in all paging levels)
 *         VPN[1] = 260-510 (in all paging levels)
 *         VPN[0] = 0       (in all paging levels)
 *
 *      In most builds, only a single 2MB "megapage" will be needed.
 *      We choose a link-time starting address of 0xffffffffeb000000
 *      within the permitted range, since the "eb" pattern is fairly
 *      distinctive and so makes it easy to visually identify any
 *      addresses originating from within iPXE's virtual address
 *      space.
 *
 *    - PTE[511] : Recursive next level page table pointer
 *
 *      This is a non-leaf PTE that points back to the page table
 *      itself.  It acts as the next level page table pointer for:
 *
 *         VPN[4] = 511 (in Sv57)
 *         VPN[3] = 511 (in Sv57 and Sv48)
 *         VPN[2] = 511 (in Sv57, Sv48, and Sv39)
 *
 *      This recursive usage creates some duplicate mappings within
 *      unused portions of the virtual address space, but allows us to
 *      use only a single physical 4kB page table.
 */

/** SBI base extension */
#define SBI_BASE 0x10
#define SBI_BASE_MVENDORID 0x04

/** Non-standard T-Head page table entry additional flags
 *
 * T-Head processors such as the C910 use the high bits of the PTE in
 * a very non-standard way that is incompatible with the RISC-V
 * specification.
 *
 * As per the "Memory Attribute Extension (XTheadMae)", bits 62 and 61
 * represent cacheability and "bufferability" (i.e. write-back
 * cacheability) respectively.  If we do not enable these bits, then
 * the processor gets incredibly confused at the point that paging is
 * enabled.  The symptom is that cache lines will occasionally fail to
 * fill, and so reads from any address may return unrelated data from
 * a previously read cache line for a different address.
 */
#define THEAD_PTE_MAEE ( 0x60 << ( __riscv_xlen - 8 ) )

/** T-Head vendor ID */
#define THEAD_MVENDORID 0x5b7

/** T-Head "sxstatus" CSR */
#define THEAD_CSR_SXSTATUS 0x5c0
#define THEAD_CSR_SXSTATUS_MAEE	0x00200000	/**< XTheadMae enabled */

	.section ".prefix.enable_paging_64", "ax", @progbits
enable_paging_64:
	/* Register usage:
	 *
	 * tp - return value (virtual address offset)
	 * a0 - page table base address
	 * a1 - currently attempted paging level
	 * a2 - enabled paging level
	 * a3 - PTE pointer
	 * a4 - PTE stride
	 * a5 - size of accessible physical address space
	 */
	progress " paging:"

	/* Calculate virtual address offset */
	LOADN	t0, prefix_link
	la	t1, _prefix
	sub	tp, t1, t0

	/* Zero PTE[0-511] */
	li	t0, PTE_COUNT
	mv	a3, a0
1:	STOREN	zero, (a3)
	addi	a3, a3, PTE_SIZE
	addi	t0, t0, -1
	bgtz	t0, 1b

	/* Construct PTE[511] as next level page table pointer */
	srli	t0, a0, PTE_PPN_SHIFT
	ori	t0, t0, PTE_V
	STOREN	t0, -PTE_SIZE(a3)

	/* Construct base page table entry for address zero */
	li	t0, PTE_LEAF
	STOREN	t0, (a0)

	/* Check for broken T-Head paging extensions */
	mv	a3, a0
	li	a7, SBI_BASE
	li	a6, SBI_BASE_MVENDORID
	ecall
	bnez	a0, 1f
	li	t0, THEAD_MVENDORID
	bne	a1, t0, 1f
	progress "thead-"
	csrr	t0, THEAD_CSR_SXSTATUS
	li	t1, THEAD_CSR_SXSTATUS_MAEE
	and	t0, t0, t1
	beqz	t0, 1f
	progress "mae-"
	LOADN	t0, (a3)
	li	t1, THEAD_PTE_MAEE
	or	t0, t0, t1
	STOREN	t0, (a3)
1:	mv	a0, a3

	/* Calculate PTE[x] address for iPXE virtual address map */
	LOADN	t0, prefix_link
	srli	t0, t0, VPN1_LSB
	andi	t0, t0, ( PTE_COUNT - 1 )
	slli	t0, t0, PTE_SIZE_LOG2
	add	a3, a0, t0

	/* Calculate PTE stride for iPXE virtual address map
	 *
	 * PPN[1] LSB is PTE bit 19 in all paging modes, and so the
	 * stride is always ( 1 << 19 )
	 */
	li	a4, 1
	slli	a4, a4, PTE_PPN1_LSB

	/* Construct PTE[x-1] for early UART, if applicable */
#ifdef EARLY_UART_REG_BASE
	li	t0, ( EARLY_UART_REG_BASE & ~( ( 1 << VPN1_LSB ) - 1 ) )
	srli	t0, t0, PTE_PPN_SHIFT
	ori	t0, t0, ( PTE_LEAF & ~PTE_X )
	STOREN	t0, -PTE_SIZE(a3)
#endif

	/* Construct PTE[x-y] for iPXE virtual address map */
	la	t0, _prefix
	srli	t0, t0, PTE_PPN_SHIFT
	LOADN	t1, (a0)
	or	t0, t0, t1
	la	t2, _ebss
	srli	t2, t2, PTE_PPN_SHIFT
1:	STOREN	t0, (a3)
	addi	a3, a3, PTE_SIZE
	add	t0, t0, a4
	ble	t0, t2, 1b

	/* Find highest supported paging level */
	li	a1, SATP_MODE_SV57
enable_paging_64_loop:

	/* Calculate PTE stride for identity map at this paging level
	 *
	 * a1 == 10 == Sv57: PPN[4] LSB is PTE bit 46  =>  stride := 1 << 46
	 * a1 ==  9 == Sv48: PPN[3] LSB is PTE bit 37  =>  stride := 1 << 37
	 * a1 ==  8 == Sv39: PPN[2] LSB is PTE bit 28  =>  stride := 1 << 28
	 *
	 * and so we calculate stride a4 := ( 1 << ( 9 * a1 - 44 ) )
	 */
	slli	a4, a1, 3
	add	a4, a4, a1
	addi	a4, a4, -44
	li	t0, 1
	sll	a4, t0, a4

	/* Calculate size of accessible physical address space
	 *
	 * The identity map comprises only the lower half of the PTEs,
	 * since virtual addresses for the higher half must have all
	 * high bits set, and so cannot form part of an identity map.
	 */
	slli	a5, a4, ( PTE_PPN_SHIFT + ( PTE_COUNT_LOG2 - 1 ) )

	/* Construct PTE[0-255] for identity map at this paging level */
	mv	a3, a0
	li	t0, ( PTE_COUNT / 2 )
	LOADN	t1, (a0)
1:	STOREN	t1, (a3)
	addi	a3, a3, PTE_SIZE
	add	t1, t1, a4
	addi	t0, t0, -1
	bgtz	t0, 1b

	/* Attempt to enable paging, and read back active paging level */
	slli	t0, a1, SATP_MODE_SHIFT
	srli	t1, a0, PAGE_SHIFT
	or	t0, t0, t1
	csrw	satp, t0
	sfence.vma
	csrr	a2, satp
	srli	a2, a2, SATP_MODE_SHIFT

	/* Loop until we successfully enable paging, or run out of levels */
	beq	a2, a1, 1f
	csrw	satp, zero
	addi	a1, a1, -1
	li	t0, SATP_MODE_SV39
	bge	a1, t0, enable_paging_64_loop
	mv	tp, zero
	mv	a5, zero
1:
	/* Adjust return address to a virtual address */
	sub	ra, ra, tp

	/* Return, with or without paging enabled */
	paging_mode_name a2
	mv	a0, a5
	ret
	.size	enable_paging_64, . - enable_paging_64

	/* Convert 64-bit physical address to virtual address */
	.macro	phys_to_virt_64 rd, rs:vararg
	.ifnb	\rs
	mv	\rd, \rs
	.endif
	.endm

	/* Early UART base address when 64-bit paging is enabled
	 *
	 * When an early UART is in use, we choose to use the 2MB
	 * "megapage" immediately below iPXE itself to map the UART.
	 */
#ifdef EARLY_UART_REG_BASE
	.section ".rodata.early_uart_reg_base_64_virt", "a", @progbits
	.balign	8
early_uart_reg_base_64_virt:
	.dword	( _base - ( 1 << VPN1_LSB ) + \
	          ( EARLY_UART_REG_BASE & ( ( 1 << VPN1_LSB ) - 1 ) ) )
	.size	early_uart_reg_base_64_virt, . - early_uart_reg_base_64_virt
#endif

/*****************************************************************************
 *
 * Disable 64-bit paging
 *
 *****************************************************************************
 *
 * This function may be called with either virtual or flat physical
 * addressing.  It does not require a valid stack pointer.
 *
 * Parameters:
 *
 *   tp - Virtual address offset
 *
 * Returns:
 *
 *   tp - Virtual address offset (zeroed)
 *   pc - Updated to a physical address
 *
 */

	.section ".prefix.disable_paging_64", "ax", @progbits
disable_paging_64:
	/* Register usage:
	 *
	 * tp - virtual address offset
	 */

	/* Jump to physical address */
	la	t0, 1f
	bgez	t0, 1f
	add	t0, t0, tp
	jr	t0
1:
	/* Disable paging */
	csrw	satp, zero
	sfence.vma

	/* Update return address to a physical address */
	bgez	ra, 1f
	add	ra, ra, tp
1:
	/* Return with paging disabled and virtual offset zeroed */
	mv	tp, zero
	ret
	.size	disable_paging_64, . - disable_paging_64

/*****************************************************************************
 *
 * Enable 32-bit paging
 *
 *****************************************************************************
 *
 * Construct a 32-bit page table to map the whole of the 32-bit
 * address space with a fixed offset selected to map iPXE itself at
 * its link-time address (which must be 4MB-aligned).
 *
 * This function must be called with flat physical addressing.  It
 * does not require a valid stack pointer.
 *
 * Parameters:
 *
 *   a0 - Page table to fill in (4kB, must be aligned to a 4kB boundary)
 *
 * Returns:
 *
 *   a0 - Size of accessible physical address space (or zero for no limit)
 *   tp - Virtual address offset
 *   pc - Updated to a virtual address if paging enabled
 *
 * A 4kB 32-bit page table contains 1024 4-byte PTEs.  We choose to
 * use these to produce a circular map of the 32-bit address space
 * using 4MB "megapages", with a fixed offset to align the virtual and
 * link-time addresses.
 *
 * To handle the transition from physical to virtual addresses, we
 * temporarily adjust the PTE covering the current program counter to
 * be a direct physical map (so that the program counter remains valid
 * at the moment when paging is enabled), then jump to a virtual
 * address, then restore the temporarily modified PTE.
 */

	.equ	enable_paging_32_xalign, 32

	.section ".prefix.enable_paging_32", "ax", @progbits
enable_paging_32:
	/* Register usage:
	 *
	 * tp - return value (virtual address offset)
	 * a0 - page table base address
	 * a1 - enabled paging level
	 * a2 - PTE pointer
	 * a3 - saved content of temporarily modified PTE
	 */
	progress " paging:"

	/* Calculate virtual address offset */
	LOADN	t0, prefix_link
	la	t1, _prefix
	sub	tp, t1, t0

	/* Construct PTEs for circular map */
	mv	a2, a0
	li	t0, PTE_COUNT
	mv	t1, tp
	ori	t1, t1, ( PTE_LEAF << PTE_PPN_SHIFT )
	li	t2, ( 1 << ( PTE_PPN1_LSB + PTE_PPN_SHIFT ) )
1:	srli	t3, t1, PTE_PPN_SHIFT
	STOREN	t3, (a2)
	addi	a2, a2, PTE_SIZE
	add	t1, t1, t2
	addi	t0, t0, -1
	bgtz	t0, 1b

	/* Temporarily modify PTE for transition code to be an identity map */
	la	t0, enable_paging_32_xstart
	srli	t0, t0, VPN1_LSB
	slli	t1, t0, PTE_SIZE_LOG2
	add	a2, a0, t1
	LOADN	a3, (a2)
	slli	t0, t0, PTE_PPN1_LSB
	ori	t0, t0, PTE_LEAF
	STOREN	t0, (a2)

	/* Adjust PTE pointer to a virtual address */
	sub	a2, a2, tp

	/* Attempt to enable paging, and read back active paging level */
	la	t0, 1f
	sub	t0, t0, tp
	li	t1, ( SATP_MODE_SV32 << SATP_MODE_SHIFT )
	srli	t2, a0, PAGE_SHIFT
	or	t1, t1, t2
	.balign	enable_paging_32_xalign
	/* Start of transition code */
enable_paging_32_xstart:
	csrw	satp, t1
	sfence.vma
	csrr	a1, satp
	beqz	a1, 2f
	jr	t0
1:	/* Restore temporarily modified PTE */
	STOREN	a3, (a2)
	sfence.vma
	/* End of transition code */
	.equ	enable_paging_32_xlen, . - enable_paging_32_xstart
2:	srli	a1, a1, SATP_MODE_SHIFT

	/* Zero SATP and virtual address offset if paging is not enabled */
	bnez	a1, 1f
	csrw	satp, zero
	mv	tp, zero
1:
	/* Adjust return address to a virtual address */
	sub	ra, ra, tp

	/* Return, with or without paging enabled */
	paging_mode_name a1
	mv	a0, zero
	ret
	.size	enable_paging_32, . - enable_paging_32

	/* Ensure that transition code did not cross an alignment boundary */
	.section ".bss.enable_paging_32_xcheck", "aw", @nobits
	.org	. + enable_paging_32_xalign - enable_paging_32_xlen

	/* Convert 32-bit physical address to virtual address */
	.macro	phys_to_virt_32 rd, rs:vararg
	.ifnb	\rs
	sub	\rd, \rs, tp
	.else
	sub	\rd, \rd, tp
	.endif
	.endm

/*****************************************************************************
 *
 * Disable 32-bit paging
 *
 *****************************************************************************
 *
 * This function may be called with either virtual or flat physical
 * addressing.  It does not require a valid stack pointer.
 *
 * Parameters:
 *
 *   tp - Virtual address offset
 *
 * Returns:
 *
 *   tp - Virtual address offset (zeroed)
 *   pc - Updated to a physical address
 *
 */

	.equ	disable_paging_32_xalign, 16

	.section ".prefix.disable_paging_32", "ax", @progbits
disable_paging_32:
	/* Register usage:
	 *
	 * tp - virtual address offset
	 * a0 - page table address
	 * a1 - transition PTE pointer
	 * a2 - transition PTE content
	 */

	/* Get page table address, and exit if paging is already disabled */
	csrr	a0, satp
	beqz	a0, 99f
	slli	a0, a0, PAGE_SHIFT
	sub	a0, a0, tp

	/* Prepare for modifying transition PTE */
	la	t0, disable_paging_32_xstart
	add	t0, t0, tp
	srli	t0, t0, VPN1_LSB
	slli	a1, t0, PTE_SIZE_LOG2
	add	a1, a1, a0
	slli	a2, t0, PTE_PPN1_LSB
	ori	a2, a2, PTE_LEAF

	/* Jump to physical address in transition PTE, and disable paging */
	la	t0, 1f
	add	t0, t0, tp
	.balign	disable_paging_32_xalign
	/* Start of transition code */
disable_paging_32_xstart:
	STOREN	a2, (a1)
	sfence.vma
	jr	t0
1:	csrw	satp, zero
	sfence.vma
	/* End of transition code */
	.equ	disable_paging_32_xlen, . - disable_paging_32_xstart

	/* Update return address to a physical address */
	add	ra, ra, tp

99:	/* Return with paging disabled and virtual offset zeroed */
	mv	tp, zero
	ret
	.size	disable_paging_32, . - disable_paging_32

	/* Ensure that transition code did not cross an alignment boundary */
	.section ".bss.disable_paging_32_xcheck", "aw", @nobits
	.org	. + disable_paging_32_xalign - disable_paging_32_xlen

/*****************************************************************************
 *
 * Poison .bss section
 *
 *****************************************************************************
 *
 * Fill the .bss section with an invalid non-zero value to expose bugs
 * in early initialisation code that erroneously relies upon variables
 * in .bss before the section has been zeroed.
 *
 * We use the value 0xeb55eb55eb55eb55 ("EBSS") since this is
 * immediately recognisable as a value in a crash dump, and will
 * trigger a page fault if dereferenced since the address is in a
 * non-canonical form.
 *
 * Poisoning the .bss will overwrite the relocation records, and so
 * can be done only as a debugging step on a system where relocation
 * is known to be unnecessary (e.g. because paging is supported).
 *
 * This function does not require a valid stack pointer, but will
 * destroy any existing stack contents if the stack happens to be
 * placed within the original .bss section.
 *
 * Parameters: none
 *
 * Returns: none
 *
 */

	.equ	poison_bss_value_32, 0xeb55eb55
	.equ	poison_bss_value_64, 0xeb55eb55eb55eb55
	.equ	poison_bss_value, _C2 ( poison_bss_value_, __riscv_xlen )

	.section ".prefix.poison_bss", "ax", @progbits
poison_bss:
	/* Fill .bss section */
	la	t0, _bss
	la	t1, _ebss
	li	t2, poison_bss_value
1:	STOREN	t2, (t0)
	addi	t0, t0, ( __riscv_xlen / 8 )
	blt	t0, t1, 1b
	ret
	.size	poison_bss, . - poison_bss

/*****************************************************************************
 *
 * Install iPXE to a suitable runtime address
 *
 *****************************************************************************
 *
 * Identify a suitable runtime address for iPXE, relocate there, and
 * set up for running normal C code.
 *
 * A valid temporary stack pointer is required.  A 4kB space for a
 * temporary page table may be provided, and must be provided if the
 * iPXE image is running from read-only memory.
 *
 * Note that this function does not preserve the callee-save registers.
 *
 * Parameters:
 *
 *   a0 - Boot hart ID
 *   a1 - Device tree physical address
 *   a2 - Optional temporary page table space (4kB, aligned to a 4kB boundary)
 *   sp - Valid temporary stack pointer
 *
 * Returns:
 *
 *   pc - Updated to be within the relocated iPXE
 *   sp - Top of internal stack
 *   tp - Virtual address offset
 *
 */

	.section ".prefix.install", "ax", @progbits
	.globl	install
install:
	/* Register usage:
	 *
	 * s0 - boot hart ID
	 * s1 - device tree physical address
	 * s2 - saved return address
	 * s3 - relocation records physical address
	 * s4 - maximum accessible physical address
	 * s5 - relocation physical address
	 * s6 - relocation offset
	 * tp - virtual address offset
	 */
	mv	tp, zero
	progress "\r\nSBI->iPXE hart:"
	print_hex_reg a0
	progress " temp:"
	print_hex_reg a2
	progress " fdt:"
	print_hex_reg a1
	progress "\r\nSBI->iPXE phys:"
	print_hex_addr _prefix
	progress " virt:"
	print_hex_data prefix_virt
	mv	s0, a0
	mv	s1, a1
	mv	s2, ra
	la	s3, _edata

	/* Poison .bss if configured to do so */
#if POISON_BSS
	call	poison_bss
#endif

	/* Attempt to enable paging, if we have temporary page table space */
	mv	a0, a2
	beqz	a2, 1f
	call	enable_paging
1:	addi	s4, a0, -1

	/* Apply relocations, if still needed after enabling paging */
	mv	a0, s3
	call	apply_relocs

	/* Find a suitable address for relocation (using temporary stack) */
	phys_to_virt a0, s1
	mv	a1, s4
	phys_to_virt sp
	call	fdtmem_relocate
	mv	s5, a0
	progress "SBI->iPXE dest:"
	print_hex_reg a0

	/* Disable paging */
	call	disable_paging

	/* Determine relocation offset */
	la	s6, _prefix
	sub	s6, s5, s6

	/* Copy iPXE image to new location and zero .bss */
	mv	t0, s5
	la	t1, _prefix
	la	t2, _edata
1:	LOADN	t3, (t1)
	STOREN	t3, (t0)
	addi	t0, t0, ( __riscv_xlen / 8 )
	addi	t1, t1, ( __riscv_xlen / 8 )
	blt	t1, t2, 1b
	la	t1, _ebss
	add	t1, t1, s6
2:	STOREN	zero, (t0)
	addi	t0, t0, ( __riscv_xlen / 8 )
	blt	t0, t1, 2b

	/* Jump to relocated copy */
	la	t0, 1f
	add	t0, t0, s6
	jr	t0
1:
	/* Attempt to re-enable paging */
	la	a0, page_table
	call	enable_paging

	/* Reapply relocations, if still needed after enabling paging */
	phys_to_virt a0, s3
	call	apply_relocs

	/* Load stack pointer */
	la	sp, _estack

	/* Store boot hart */
	STOREN	s0, boot_hart, t0

	/* Copy and register system device tree */
	phys_to_virt a0, s1
	mv	a1, s4
	call	fdtmem_register

	/* Return to a virtual address in the relocated copy */
	add	ra, s2, s6
	sub	ra, ra, tp
	progress "\r\n"
	ret
	.size	install, . - install

/*****************************************************************************
 *
 * Reset (or lock up) system
 *
 *****************************************************************************
 *
 * Reset via system via SBI, as a means of exiting from a prefix that
 * has no other defined exit path.  If the reset fails, lock up the
 * system since there is nothing else that can sensibly be done.
 *
 * This function does not require a valid stack pointer.
 *
 * Parameters: none
 *
 * Returns: n/a (does not return)
 *
 */

/* SBI system reset extension */
#define SBI_SRST ( ( 'S' << 24 ) | ( 'R' << 16 ) | ( 'S' << 8 ) | 'T' )
#define SBI_SRST_SYSTEM_RESET 0x00
#define SBI_RESET_COLD 0x00000001

/* SBI legacy shutdown */
#define SBI_LEGACY_SHUTDOWN 0x08

	.section ".prefix.reset_system", "ax", @progbits
	.globl	reset_system
reset_system:
	/* Register usage: irrelevant (does not return) */
	progress "\r\niPXE->SBI reset\r\n"

	/* Attempt reset */
	li	a7, SBI_SRST
	li	a6, SBI_SRST_SYSTEM_RESET
	li	a0, SBI_RESET_COLD
	mv	a1, zero
	ecall
	progress "(reset failed)\r\n"

	/* Attempt legacy shutdown */
	li	a7, SBI_LEGACY_SHUTDOWN
	ecall
	progress "(legacy shutdown failed)\r\n"

	/* If reset failed, lock the system */
1:	wfi
	j	1b
	.size	reset_system, . - reset_system

/*****************************************************************************
 *
 * File split information for the compressor
 *
 *****************************************************************************
 */

/* ELF machine type */
#define EM_RISCV 243

	.section ".zinfo", "a", @progbits
	.org	0
	/* Copy initialised-data portion of image */
	.ascii	"COPY"
	.word	0
	.word	_filesz
	.word	1
	/* Notify compressor of link-time base address */
	.ascii	"BASE"
	.word	0
	.dword	_base
	/* Construct compressed relocation records */
	.ascii	"ZREL"
	.word	_reloc_offset
	.word	_reloc_filesz
	.word	EM_RISCV