/*
* ARM v8.5-MemTag Operations
*
* Copyright (c) 2020 Linaro, Ltd.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*/
#include "qemu/osdep.h"
#include "cpu.h"
#include "internals.h"
#include "exec/exec-all.h"
#include "exec/ram_addr.h"
#include "exec/cpu_ldst.h"
#include "exec/helper-proto.h"
#include "qapi/error.h"
#include "qemu/guest-random.h"
static int choose_nonexcluded_tag(int tag, int offset, uint16_t exclude)
{
if (exclude == 0xffff) {
return 0;
}
if (offset == 0) {
while (exclude & (1 << tag)) {
tag = (tag + 1) & 15;
}
} else {
do {
do {
tag = (tag + 1) & 15;
} while (exclude & (1 << tag));
} while (--offset > 0);
}
return tag;
}
/**
* allocation_tag_mem:
* @env: the cpu environment
* @ptr_mmu_idx: the addressing regime to use for the virtual address
* @ptr: the virtual address for which to look up tag memory
* @ptr_access: the access to use for the virtual address
* @ptr_size: the number of bytes in the normal memory access
* @tag_access: the access to use for the tag memory
* @tag_size: the number of bytes in the tag memory access
* @ra: the return address for exception handling
*
* Our tag memory is formatted as a sequence of little-endian nibbles.
* That is, the byte at (addr >> (LOG2_TAG_GRANULE + 1)) contains two
* tags, with the tag at [3:0] for the lower addr and the tag at [7:4]
* for the higher addr.
*
* Here, resolve the physical address from the virtual address, and return
* a pointer to the corresponding tag byte. Exit with exception if the
* virtual address is not accessible for @ptr_access.
*
* The @ptr_size and @tag_size values may not have an obvious relation
* due to the alignment of @ptr, and the number of tag checks required.
*
* If there is no tag storage corresponding to @ptr, return NULL.
*/
static uint8_t *allocation_tag_mem(CPUARMState *env, int ptr_mmu_idx,
uint64_t ptr, MMUAccessType ptr_access,
int ptr_size, MMUAccessType tag_access,
int tag_size, uintptr_t ra)
{
#ifdef CONFIG_USER_ONLY
uint64_t clean_ptr = useronly_clean_ptr(ptr);
int flags = page_get_flags(clean_ptr);
uint8_t *tags;
uintptr_t index;
if (!(flags & (ptr_access == MMU_DATA_STORE ? PAGE_WRITE : PAGE_READ))) {
/* SIGSEGV */
arm_cpu_tlb_fill(env_cpu(env), ptr, ptr_size, ptr_access,
ptr_mmu_idx, false, ra);
g_assert_not_reached();
}
/* Require both MAP_ANON and PROT_MTE for the page. */
if (!(flags & PAGE_ANON) || !(flags & PAGE_MTE)) {
return NULL;
}
tags = page_get_target_data(clean_ptr);
if (tags == NULL) {
size_t alloc_size = TARGET_PAGE_SIZE >> (LOG2_TAG_GRANULE + 1);
tags = page_alloc_target_data(clean_ptr, alloc_size);
assert(tags != NULL);
}
index = extract32(ptr, LOG2_TAG_GRANULE + 1,
TARGET_PAGE_BITS - LOG2_TAG_GRANULE - 1);
return tags + index;
#else
uintptr_t index;
CPUIOTLBEntry *iotlbentry;
int in_page, flags;
ram_addr_t ptr_ra;
hwaddr ptr_paddr, tag_paddr, xlat;
MemoryRegion *mr;
ARMASIdx tag_asi;
AddressSpace *tag_as;
void *host;
/*
* Probe the first byte of the virtual address. This raises an
* exception for inaccessible pages, and resolves the virtual address
* into the softmmu tlb.
*
* When RA == 0, this is for mte_probe1. The page is expected to be
* valid. Indicate to probe_access_flags no-fault, then assert that
* we received a valid page.
*/
flags = probe_access_flags(env, ptr, ptr_access, ptr_mmu_idx,
ra == 0, &host, ra);
assert(!(flags & TLB_INVALID_MASK));
/*
* Find the iotlbentry for ptr. This *must* be present in the TLB
* because we just found the mapping.
* TODO: Perhaps there should be a cputlb helper that returns a
* matching tlb entry + iotlb entry.
*/
index = tlb_index(env, ptr_mmu_idx, ptr);
# ifdef CONFIG_DEBUG_TCG
{
CPUTLBEntry *entry = tlb_entry(env, ptr_mmu_idx, ptr);
target_ulong comparator = (ptr_access == MMU_DATA_LOAD
? entry->addr_read
: tlb_addr_write(entry));
g_assert(tlb_hit(comparator, ptr));
}
# endif
iotlbentry = &env_tlb(env)->d[ptr_mmu_idx].iotlb[index];
/* If the virtual page MemAttr != Tagged, access unchecked. */
if (!arm_tlb_mte_tagged(&iotlbentry->attrs)) {
return NULL;
}
/*
* If not backed by host ram, there is no tag storage: access unchecked.
* This is probably a guest os bug though, so log it.
*/
if (unlikely(flags & TLB_MMIO)) {
qemu_log_mask(LOG_GUEST_ERROR,
"Page @ 0x%" PRIx64 " indicates Tagged Normal memory "
"but is not backed by host ram\n", ptr);
return NULL;
}
/*
* The Normal memory access can extend to the next page. E.g. a single
* 8-byte access to the last byte of a page will check only the last
* tag on the first page.
* Any page access exception has priority over tag check exception.
*/
in_page = -(ptr | TARGET_PAGE_MASK);
if (unlikely(ptr_size > in_page)) {
void *ignore;
flags |= probe_access_flags(env, ptr + in_page, ptr_access,
ptr_mmu_idx, ra == 0, &ignore, ra);
assert(!(flags & TLB_INVALID_MASK));
}
/* Any debug exception has priority over a tag check exception. */
if (unlikely(flags & TLB_WATCHPOINT)) {
int wp = ptr_access == MMU_DATA_LOAD ? BP_MEM_READ : BP_MEM_WRITE;
assert(ra != 0);
cpu_check_watchpoint(env_cpu(env), ptr, ptr_size,
iotlbentry->attrs, wp, ra);
}
/*
* Find the physical address within the normal mem space.
* The memory region lookup must succeed because TLB_MMIO was
* not set in the cputlb lookup above.
*/
mr = memory_region_from_host(host, &ptr_ra);
tcg_debug_assert(mr != NULL);
tcg_debug_assert(memory_region_is_ram(mr));
ptr_paddr = ptr_ra;
do {
ptr_paddr += mr->addr;
mr = mr->container;
} while (mr);
/* Convert to the physical address in tag space. */
tag_paddr = ptr_paddr >> (LOG2_TAG_GRANULE + 1);
/* Look up the address in tag space. */
tag_asi = iotlbentry->attrs.secure ? ARMASIdx_TagS : ARMASIdx_TagNS;
tag_as = cpu_get_address_space(env_cpu(env), tag_asi);
mr = address_space_translate(tag_as, tag_paddr, &xlat, NULL,
tag_access == MMU_DATA_STORE,
iotlbentry->attrs);
/*
* Note that @mr will never be NULL. If there is nothing in the address
* space at @tag_paddr, the translation will return the unallocated memory
* region. For our purposes, the result must be ram.
*/
if (unlikely(!memory_region_is_ram(mr))) {
/* ??? Failure is a board configuration error. */
qemu_log_mask(LOG_UNIMP,
"Tag Memory @ 0x%" HWADDR_PRIx " not found for "
"Normal Memory @ 0x%" HWADDR_PRIx "\n",
tag_paddr, ptr_paddr);
return NULL;
}
/*
* Ensure the tag memory is dirty on write, for migration.
* Tag memory can never contain code or display memory (vga).
*/
if (tag_access == MMU_DATA_STORE) {
ram_addr_t tag_ra = memory_region_get_ram_addr(mr) + xlat;
cpu_physical_memory_set_dirty_flag(tag_ra, DIRTY_MEMORY_MIGRATION);
}
return memory_region_get_ram_ptr(mr) + xlat;
#endif
}
uint64_t HELPER(irg)(CPUARMState *env, uint64_t rn, uint64_t rm)
{
uint16_t exclude = extract32(rm | env->cp15.gcr_el1, 0, 16);
int rrnd = extract32(env->cp15.gcr_el1, 16, 1);
int start = extract32(env->cp15.rgsr_el1, 0, 4);
int seed = extract32(env->cp15.rgsr_el1, 8, 16);
int offset, i, rtag;
/*
* Our IMPDEF choice for GCR_EL1.RRND==1 is to continue to use the
* deterministic algorithm. Except that with RRND==1 the kernel is
* not required to have set RGSR_EL1.SEED != 0, which is required for
* the deterministic algorithm to function. So we force a non-zero
* SEED for that case.
*/
if (unlikely(seed == 0) && rrnd) {
do {
Error *err = NULL;
uint16_t two;
if (qemu_guest_getrandom(&two, sizeof(two), &err) < 0) {
/*
* Failed, for unknown reasons in the crypto subsystem.
* Best we can do is log the reason and use a constant seed.
*/
qemu_log_mask(LOG_UNIMP, "IRG: Crypto failure: %s\n",
error_get_pretty(err));
error_free(err);
two = 1;
}
seed = two;
} while (seed == 0);
}
/* RandomTag */
for (i = offset = 0; i < 4; ++i) {
/* NextRandomTagBit */
int top = (extract32(seed, 5, 1) ^ extract32(seed, 3, 1) ^
extract32(seed, 2, 1) ^ extract32(seed, 0, 1));
seed = (top << 15) | (seed >> 1);
offset |= top << i;
}
rtag = choose_nonexcluded_tag(start, offset, exclude);
env->cp15.rgsr_el1 = rtag | (seed << 8);
return address_with_allocation_tag(rn, rtag);
}
uint64_t HELPER(addsubg)(CPUARMState *env, uint64_t ptr,
int32_t offset, uint32_t tag_offset)
{
int start_tag = allocation_tag_from_addr(ptr);
uint16_t exclude = extract32(env->cp15.gcr_el1, 0, 16);
int rtag = choose_nonexcluded_tag(start_tag, tag_offset, exclude);
return address_with_allocation_tag(ptr + offset, rtag);
}
static int load_tag1(uint64_t ptr, uint8_t *mem)
{
int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
return extract32(*mem, ofs, 4);
}
uint64_t HELPER(ldg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
{
int mmu_idx = cpu_mmu_index(env, false);
uint8_t *mem;
int rtag = 0;
/* Trap if accessing an invalid page. */
mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD, 1,
MMU_DATA_LOAD, 1, GETPC());
/* Load if page supports tags. */
if (mem) {
rtag = load_tag1(ptr, mem);
}
return address_with_allocation_tag(xt, rtag);
}
static void check_tag_aligned(CPUARMState *env, uint64_t ptr, uintptr_t ra)
{
if (unlikely(!QEMU_IS_ALIGNED(ptr, TAG_GRANULE))) {
arm_cpu_do_unaligned_access(env_cpu(env), ptr, MMU_DATA_STORE,
cpu_mmu_index(env, false), ra);
g_assert_not_reached();
}
}
/* For use in a non-parallel context, store to the given nibble. */
static void store_tag1(uint64_t ptr, uint8_t *mem, int tag)
{
int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
*mem = deposit32(*mem, ofs, 4, tag);
}
/* For use in a parallel context, atomically store to the given nibble. */
static void store_tag1_parallel(uint64_t ptr, uint8_t *mem, int tag)
{
int ofs = extract32(ptr, LOG2_TAG_GRANULE, 1) * 4;
uint8_t old = qatomic_read(mem);
while (1) {
uint8_t new = deposit32(old, ofs, 4, tag);
uint8_t cmp = qatomic_cmpxchg(mem, old, new);
if (likely(cmp == old)) {
return;
}
old = cmp;
}
}
typedef void stg_store1(uint64_t, uint8_t *, int);
static inline void do_stg(CPUARMState *env, uint64_t ptr, uint64_t xt,
uintptr_t ra, stg_store1 store1)
{
int mmu_idx = cpu_mmu_index(env, false);
uint8_t *mem;
check_tag_aligned(env, ptr, ra);
/* Trap if accessing an invalid page. */
mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, TAG_GRANULE,
MMU_DATA_STORE, 1, ra);
/* Store if page supports tags. */
if (mem) {
store1(ptr, mem, allocation_tag_from_addr(xt));
}
}
void HELPER(stg)(CPUARMState *env, uint64_t ptr, uint64_t xt)
{
do_stg(env, ptr, xt, GETPC(), store_tag1);
}
void HELPER(stg_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
{
do_stg(env, ptr, xt, GETPC(), store_tag1_parallel);
}
void HELPER(stg_stub)(CPUARMState *env, uint64_t ptr)
{
int mmu_idx = cpu_mmu_index(env, false);
uintptr_t ra = GETPC();
check_tag_aligned(env, ptr, ra);
probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
}
static inline void do_st2g(CPUARMState *env, uint64_t ptr, uint64_t xt,
uintptr_t ra, stg_store1 store1)
{
int mmu_idx = cpu_mmu_index(env, false);
int tag = allocation_tag_from_addr(xt);
uint8_t *mem1, *mem2;
check_tag_aligned(env, ptr, ra);
/*
* Trap if accessing an invalid page(s).
* This takes priority over !allocation_tag_access_enabled.
*/
if (ptr & TAG_GRANULE) {
/* Two stores unaligned mod TAG_GRANULE*2 -- modify two bytes. */
mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
TAG_GRANULE, MMU_DATA_STORE, 1, ra);
mem2 = allocation_tag_mem(env, mmu_idx, ptr + TAG_GRANULE,
MMU_DATA_STORE, TAG_GRANULE,
MMU_DATA_STORE, 1, ra);
/* Store if page(s) support tags. */
if (mem1) {
store1(TAG_GRANULE, mem1, tag);
}
if (mem2) {
store1(0, mem2, tag);
}
} else {
/* Two stores aligned mod TAG_GRANULE*2 -- modify one byte. */
mem1 = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
2 * TAG_GRANULE, MMU_DATA_STORE, 1, ra);
if (mem1) {
tag |= tag << 4;
qatomic_set(mem1, tag);
}
}
}
void HELPER(st2g)(CPUARMState *env, uint64_t ptr, uint64_t xt)
{
do_st2g(env, ptr, xt, GETPC(), store_tag1);
}
void HELPER(st2g_parallel)(CPUARMState *env, uint64_t ptr, uint64_t xt)
{
do_st2g(env, ptr, xt, GETPC(), store_tag1_parallel);
}
void HELPER(st2g_stub)(CPUARMState *env, uint64_t ptr)
{
int mmu_idx = cpu_mmu_index(env, false);
uintptr_t ra = GETPC();
int in_page = -(ptr | TARGET_PAGE_MASK);
check_tag_aligned(env, ptr, ra);
if (likely(in_page >= 2 * TAG_GRANULE)) {
probe_write(env, ptr, 2 * TAG_GRANULE, mmu_idx, ra);
} else {
probe_write(env, ptr, TAG_GRANULE, mmu_idx, ra);
probe_write(env, ptr + TAG_GRANULE, TAG_GRANULE, mmu_idx, ra);
}
}
#define LDGM_STGM_SIZE (4 << GMID_EL1_BS)
uint64_t HELPER(ldgm)(CPUARMState *env, uint64_t ptr)
{
int mmu_idx = cpu_mmu_index(env, false);
uintptr_t ra = GETPC();
void *tag_mem;
ptr = QEMU_ALIGN_DOWN(ptr, LDGM_STGM_SIZE);
/* Trap if accessing an invalid page. */
tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_LOAD,
LDGM_STGM_SIZE, MMU_DATA_LOAD,
LDGM_STGM_SIZE / (2 * TAG_GRANULE), ra);
/* The tag is squashed to zero if the page does not support tags. */
if (!tag_mem) {
return 0;
}
QEMU_BUILD_BUG_ON(GMID_EL1_BS != 6);
/*
* We are loading 64-bits worth of tags. The ordering of elements
* within the word corresponds to a 64-bit little-endian operation.
*/
return ldq_le_p(tag_mem);
}
void HELPER(stgm)(CPUARMState *env, uint64_t ptr, uint64_t val)
{
int mmu_idx = cpu_mmu_index(env, false);
uintptr_t ra = GETPC();
void *tag_mem;
ptr = QEMU_ALIGN_DOWN(ptr, LDGM_STGM_SIZE);
/* Trap if accessing an invalid page. */
tag_mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE,
LDGM_STGM_SIZE, MMU_DATA_LOAD,
LDGM_STGM_SIZE / (2 * TAG_GRANULE), ra);
/*
* Tag store only happens if the page support tags,
* and if the OS has enabled access to the tags.
*/
if (!tag_mem) {
return;
}
QEMU_BUILD_BUG_ON(GMID_EL1_BS != 6);
/*
* We are storing 64-bits worth of tags. The ordering of elements
* within the word corresponds to a 64-bit little-endian operation.
*/
stq_le_p(tag_mem, val);
}
void HELPER(stzgm_tags)(CPUARMState *env, uint64_t ptr, uint64_t val)
{
uintptr_t ra = GETPC();
int mmu_idx = cpu_mmu_index(env, false);
int log2_dcz_bytes, log2_tag_bytes;
intptr_t dcz_bytes, tag_bytes;
uint8_t *mem;
/*
* In arm_cpu_realizefn, we assert that dcz > LOG2_TAG_GRANULE+1,
* i.e. 32 bytes, which is an unreasonably small dcz anyway,
* to make sure that we can access one complete tag byte here.
*/
log2_dcz_bytes = env_archcpu(env)->dcz_blocksize + 2;
log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
tag_bytes = (intptr_t)1 << log2_tag_bytes;
ptr &= -dcz_bytes;
mem = allocation_tag_mem(env, mmu_idx, ptr, MMU_DATA_STORE, dcz_bytes,
MMU_DATA_STORE, tag_bytes, ra);
if (mem) {
int tag_pair = (val & 0xf) * 0x11;
memset(mem, tag_pair, tag_bytes);
}
}
/* Record a tag check failure. */
static void mte_check_fail(CPUARMState *env, uint32_t desc,
uint64_t dirty_ptr, uintptr_t ra)
{
int mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
ARMMMUIdx arm_mmu_idx = core_to_aa64_mmu_idx(mmu_idx);
int el, reg_el, tcf, select, is_write, syn;
uint64_t sctlr;
reg_el = regime_el(env, arm_mmu_idx);
sctlr = env->cp15.sctlr_el[reg_el];
el = arm_current_el(env);
if (el == 0) {
tcf = extract64(sctlr, 38, 2);
} else {
tcf = extract64(sctlr, 40, 2);
}
switch (tcf) {
case 1:
/*
* Tag check fail causes a synchronous exception.
*
* In restore_state_to_opc, we set the exception syndrome
* for the load or store operation. Unwind first so we
* may overwrite that with the syndrome for the tag check.
*/
cpu_restore_state(env_cpu(env), ra, true);
env->exception.vaddress = dirty_ptr;
is_write = FIELD_EX32(desc, MTEDESC, WRITE);
syn = syn_data_abort_no_iss(el != 0, 0, 0, 0, 0, is_write, 0x11);
raise_exception(env, EXCP_DATA_ABORT, syn, exception_target_el(env));
/* noreturn, but fall through to the assert anyway */
case 0:
/*
* Tag check fail does not affect the PE.
* We eliminate this case by not setting MTE_ACTIVE
* in tb_flags, so that we never make this runtime call.
*/
g_assert_not_reached();
case 2:
/* Tag check fail causes asynchronous flag set. */
if (regime_has_2_ranges(arm_mmu_idx)) {
select = extract64(dirty_ptr, 55, 1);
} else {
select = 0;
}
env->cp15.tfsr_el[el] |= 1 << select;
#ifdef CONFIG_USER_ONLY
/*
* Stand in for a timer irq, setting _TIF_MTE_ASYNC_FAULT,
* which then sends a SIGSEGV when the thread is next scheduled.
* This cpu will return to the main loop at the end of the TB,
* which is rather sooner than "normal". But the alternative
* is waiting until the next syscall.
*/
qemu_cpu_kick(env_cpu(env));
#endif
break;
default:
/* Case 3: Reserved. */
qemu_log_mask(LOG_GUEST_ERROR,
"Tag check failure with SCTLR_EL%d.TCF%s "
"set to reserved value %d\n",
reg_el, el ? "" : "0", tcf);
break;
}
}
/*
* Perform an MTE checked access for a single logical or atomic access.
*/
static bool mte_probe1_int(CPUARMState *env, uint32_t desc, uint64_t ptr,
uintptr_t ra, int bit55)
{
int mem_tag, mmu_idx, ptr_tag, size;
MMUAccessType type;
uint8_t *mem;
ptr_tag = allocation_tag_from_addr(ptr);
if (tcma_check(desc, bit55, ptr_tag)) {
return true;
}
mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
size = FIELD_EX32(desc, MTEDESC, ESIZE);
mem = allocation_tag_mem(env, mmu_idx, ptr, type, size,
MMU_DATA_LOAD, 1, ra);
if (!mem) {
return true;
}
mem_tag = load_tag1(ptr, mem);
return ptr_tag == mem_tag;
}
/*
* No-fault version of mte_check1, to be used by SVE for MemSingleNF.
* Returns false if the access is Checked and the check failed. This
* is only intended to probe the tag -- the validity of the page must
* be checked beforehand.
*/
bool mte_probe1(CPUARMState *env, uint32_t desc, uint64_t ptr)
{
int bit55 = extract64(ptr, 55, 1);
/* If TBI is disabled, the access is unchecked. */
if (unlikely(!tbi_check(desc, bit55))) {
return true;
}
return mte_probe1_int(env, desc, ptr, 0, bit55);
}
uint64_t mte_check1(CPUARMState *env, uint32_t desc,
uint64_t ptr, uintptr_t ra)
{
int bit55 = extract64(ptr, 55, 1);
/* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
if (unlikely(!tbi_check(desc, bit55))) {
return ptr;
}
if (unlikely(!mte_probe1_int(env, desc, ptr, ra, bit55))) {
mte_check_fail(env, desc, ptr, ra);
}
return useronly_clean_ptr(ptr);
}
uint64_t HELPER(mte_check1)(CPUARMState *env, uint32_t desc, uint64_t ptr)
{
return mte_check1(env, desc, ptr, GETPC());
}
/*
* Perform an MTE checked access for multiple logical accesses.
*/
/**
* checkN:
* @tag: tag memory to test
* @odd: true to begin testing at tags at odd nibble
* @cmp: the tag to compare against
* @count: number of tags to test
*
* Return the number of successful tests.
* Thus a return value < @count indicates a failure.
*
* A note about sizes: count is expected to be small.
*
* The most common use will be LDP/STP of two integer registers,
* which means 16 bytes of memory touching at most 2 tags, but
* often the access is aligned and thus just 1 tag.
*
* Using AdvSIMD LD/ST (multiple), one can access 64 bytes of memory,
* touching at most 5 tags. SVE LDR/STR (vector) with the default
* vector length is also 64 bytes; the maximum architectural length
* is 256 bytes touching at most 9 tags.
*
* The loop below uses 7 logical operations and 1 memory operation
* per tag pair. An implementation that loads an aligned word and
* uses masking to ignore adjacent tags requires 18 logical operations
* and thus does not begin to pay off until 6 tags.
* Which, according to the survey above, is unlikely to be common.
*/
static int checkN(uint8_t *mem, int odd, int cmp, int count)
{
int n = 0, diff;
/* Replicate the test tag and compare. */
cmp *= 0x11;
diff = *mem++ ^ cmp;
if (odd) {
goto start_odd;
}
while (1) {
/* Test even tag. */
if (unlikely((diff) & 0x0f)) {
break;
}
if (++n == count) {
break;
}
start_odd:
/* Test odd tag. */
if (unlikely((diff) & 0xf0)) {
break;
}
if (++n == count) {
break;
}
diff = *mem++ ^ cmp;
}
return n;
}
uint64_t mte_checkN(CPUARMState *env, uint32_t desc,
uint64_t ptr, uintptr_t ra)
{
int mmu_idx, ptr_tag, bit55;
uint64_t ptr_last, ptr_end, prev_page, next_page;
uint64_t tag_first, tag_end;
uint64_t tag_byte_first, tag_byte_end;
uint32_t esize, total, tag_count, tag_size, n, c;
uint8_t *mem1, *mem2;
MMUAccessType type;
bit55 = extract64(ptr, 55, 1);
/* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
if (unlikely(!tbi_check(desc, bit55))) {
return ptr;
}
ptr_tag = allocation_tag_from_addr(ptr);
if (tcma_check(desc, bit55, ptr_tag)) {
goto done;
}
mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
type = FIELD_EX32(desc, MTEDESC, WRITE) ? MMU_DATA_STORE : MMU_DATA_LOAD;
esize = FIELD_EX32(desc, MTEDESC, ESIZE);
total = FIELD_EX32(desc, MTEDESC, TSIZE);
/* Find the addr of the end of the access, and of the last element. */
ptr_end = ptr + total;
ptr_last = ptr_end - esize;
/* Round the bounds to the tag granule, and compute the number of tags. */
tag_first = QEMU_ALIGN_DOWN(ptr, TAG_GRANULE);
tag_end = QEMU_ALIGN_UP(ptr_last, TAG_GRANULE);
tag_count = (tag_end - tag_first) / TAG_GRANULE;
/* Round the bounds to twice the tag granule, and compute the bytes. */
tag_byte_first = QEMU_ALIGN_DOWN(ptr, 2 * TAG_GRANULE);
tag_byte_end = QEMU_ALIGN_UP(ptr_last, 2 * TAG_GRANULE);
/* Locate the page boundaries. */
prev_page = ptr & TARGET_PAGE_MASK;
next_page = prev_page + TARGET_PAGE_SIZE;
if (likely(tag_end - prev_page <= TARGET_PAGE_SIZE)) {
/* Memory access stays on one page. */
tag_size = (tag_byte_end - tag_byte_first) / (2 * TAG_GRANULE);
mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, total,
MMU_DATA_LOAD, tag_size, ra);
if (!mem1) {
goto done;
}
/* Perform all of the comparisons. */
n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, tag_count);
} else {
/* Memory access crosses to next page. */
tag_size = (next_page - tag_byte_first) / (2 * TAG_GRANULE);
mem1 = allocation_tag_mem(env, mmu_idx, ptr, type, next_page - ptr,
MMU_DATA_LOAD, tag_size, ra);
tag_size = (tag_byte_end - next_page) / (2 * TAG_GRANULE);
mem2 = allocation_tag_mem(env, mmu_idx, next_page, type,
ptr_end - next_page,
MMU_DATA_LOAD, tag_size, ra);
/*
* Perform all of the comparisons.
* Note the possible but unlikely case of the operation spanning
* two pages that do not both have tagging enabled.
*/
n = c = (next_page - tag_first) / TAG_GRANULE;
if (mem1) {
n = checkN(mem1, ptr & TAG_GRANULE, ptr_tag, c);
}
if (n == c) {
if (!mem2) {
goto done;
}
n += checkN(mem2, 0, ptr_tag, tag_count - c);
}
}
/*
* If we failed, we know which granule. Compute the element that
* is first in that granule, and signal failure on that element.
*/
if (unlikely(n < tag_count)) {
uint64_t fail_ofs;
fail_ofs = tag_first + n * TAG_GRANULE - ptr;
fail_ofs = ROUND_UP(fail_ofs, esize);
mte_check_fail(env, desc, ptr + fail_ofs, ra);
}
done:
return useronly_clean_ptr(ptr);
}
uint64_t HELPER(mte_checkN)(CPUARMState *env, uint32_t desc, uint64_t ptr)
{
return mte_checkN(env, desc, ptr, GETPC());
}
/*
* Perform an MTE checked access for DC_ZVA.
*/
uint64_t HELPER(mte_check_zva)(CPUARMState *env, uint32_t desc, uint64_t ptr)
{
uintptr_t ra = GETPC();
int log2_dcz_bytes, log2_tag_bytes;
int mmu_idx, bit55;
intptr_t dcz_bytes, tag_bytes, i;
void *mem;
uint64_t ptr_tag, mem_tag, align_ptr;
bit55 = extract64(ptr, 55, 1);
/* If TBI is disabled, the access is unchecked, and ptr is not dirty. */
if (unlikely(!tbi_check(desc, bit55))) {
return ptr;
}
ptr_tag = allocation_tag_from_addr(ptr);
if (tcma_check(desc, bit55, ptr_tag)) {
goto done;
}
/*
* In arm_cpu_realizefn, we asserted that dcz > LOG2_TAG_GRANULE+1,
* i.e. 32 bytes, which is an unreasonably small dcz anyway, to make
* sure that we can access one complete tag byte here.
*/
log2_dcz_bytes = env_archcpu(env)->dcz_blocksize + 2;
log2_tag_bytes = log2_dcz_bytes - (LOG2_TAG_GRANULE + 1);
dcz_bytes = (intptr_t)1 << log2_dcz_bytes;
tag_bytes = (intptr_t)1 << log2_tag_bytes;
align_ptr = ptr & -dcz_bytes;
/*
* Trap if accessing an invalid page. DC_ZVA requires that we supply
* the original pointer for an invalid page. But watchpoints require
* that we probe the actual space. So do both.
*/
mmu_idx = FIELD_EX32(desc, MTEDESC, MIDX);
(void) probe_write(env, ptr, 1, mmu_idx, ra);
mem = allocation_tag_mem(env, mmu_idx, align_ptr, MMU_DATA_STORE,
dcz_bytes, MMU_DATA_LOAD, tag_bytes, ra);
if (!mem) {
goto done;
}
/*
* Unlike the reasoning for checkN, DC_ZVA is always aligned, and thus
* it is quite easy to perform all of the comparisons at once without
* any extra masking.
*
* The most common zva block size is 64; some of the thunderx cpus use
* a block size of 128. For user-only, aarch64_max_initfn will set the
* block size to 512. Fill out the other cases for future-proofing.
*
* In order to be able to find the first miscompare later, we want the
* tag bytes to be in little-endian order.
*/
switch (log2_tag_bytes) {
case 0: /* zva_blocksize 32 */
mem_tag = *(uint8_t *)mem;
ptr_tag *= 0x11u;
break;
case 1: /* zva_blocksize 64 */
mem_tag = cpu_to_le16(*(uint16_t *)mem);
ptr_tag *= 0x1111u;
break;
case 2: /* zva_blocksize 128 */
mem_tag = cpu_to_le32(*(uint32_t *)mem);
ptr_tag *= 0x11111111u;
break;
case 3: /* zva_blocksize 256 */
mem_tag = cpu_to_le64(*(uint64_t *)mem);
ptr_tag *= 0x1111111111111111ull;
break;
default: /* zva_blocksize 512, 1024, 2048 */
ptr_tag *= 0x1111111111111111ull;
i = 0;
do {
mem_tag = cpu_to_le64(*(uint64_t *)(mem + i));
if (unlikely(mem_tag != ptr_tag)) {
goto fail;
}
i += 8;
align_ptr += 16 * TAG_GRANULE;
} while (i < tag_bytes);
goto done;
}
if (likely(mem_tag == ptr_tag)) {
goto done;
}
fail:
/* Locate the first nibble that differs. */
i = ctz64(mem_tag ^ ptr_tag) >> 4;
mte_check_fail(env, desc, align_ptr + i * TAG_GRANULE, ra);
done:
return useronly_clean_ptr(ptr);
}
