#include "qemu/osdep.h"
#include "qemu/cutils.h"
#include "qemu/memalign.h"
#include "cpu.h"
#include "helper_regs.h"
#include "hw/ppc/spapr.h"
#include "mmu-hash64.h"
#include "mmu-book3s-v3.h"
static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
{
/*
* hash value/pteg group index is normalized by HPT mask
*/
if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
return false;
}
return true;
}
static target_ulong h_enter(PowerPCCPU *cpu, SpaprMachineState *spapr,
target_ulong opcode, target_ulong *args)
{
target_ulong flags = args[0];
target_ulong ptex = args[1];
target_ulong pteh = args[2];
target_ulong ptel = args[3];
unsigned apshift;
target_ulong raddr;
target_ulong slot;
const ppc_hash_pte64_t *hptes;
apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
if (!apshift) {
/* Bad page size encoding */
return H_PARAMETER;
}
raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
if (is_ram_address(spapr, raddr)) {
/* Regular RAM - should have WIMG=0010 */
if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
return H_PARAMETER;
}
} else {
target_ulong wimg_flags;
/* Looks like an IO address */
/* FIXME: What WIMG combinations could be sensible for IO?
* For now we allow WIMG=010x, but are there others? */
/* FIXME: Should we check against registered IO addresses? */
wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
if (wimg_flags != HPTE64_R_I &&
wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
return H_PARAMETER;
}
}
pteh &= ~0x60ULL;
if (!valid_ptex(cpu, ptex)) {
return H_PARAMETER;
}
slot = ptex & 7ULL;
ptex = ptex & ~7ULL;
if (likely((flags & H_EXACT) == 0)) {
hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
for (slot = 0; slot < 8; slot++) {
if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
break;
}
}
ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
if (slot == 8) {
return H_PTEG_FULL;
}
} else {
hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
return H_PTEG_FULL;
}
ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
}
spapr_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
args[0] = ptex + slot;
return H_SUCCESS;
}
typedef enum {
REMOVE_SUCCESS = 0,
REMOVE_NOT_FOUND = 1,
REMOVE_PARM = 2,
REMOVE_HW = 3,
} RemoveResult;
static RemoveResult remove_hpte(PowerPCCPU *cpu
, target_ulong ptex,
target_ulong avpn,
target_ulong flags,
target_ulong *vp, target_ulong *rp)
{
const ppc_hash_pte64_t *hptes;
target_ulong v, r;
if (!valid_ptex(cpu, ptex)) {
return REMOVE_PARM;
}
hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
v = ppc_hash64_hpte0(cpu, hptes, 0);
r = ppc_hash64_hpte1(cpu, hptes, 0);
ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
if ((v & HPTE64_V_VALID) == 0 ||
((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
((flags & H_ANDCOND) && (v & avpn) != 0)) {
return REMOVE_NOT_FOUND;
}
*vp = v;
*rp = r;
spapr_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
return REMOVE_SUCCESS;
}
static target_ulong h_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
target_ulong opcode, target_ulong *args)
{
CPUPPCState *env = &cpu->env;
target_ulong flags = args[0];
target_ulong ptex = args[1];
target_ulong avpn = args[2];
RemoveResult ret;
ret = remove_hpte(cpu, ptex, avpn, flags,
&args[0], &args[1]);
switch (ret) {
case REMOVE_SUCCESS:
check_tlb_flush(env, true);
return H_SUCCESS;
case REMOVE_NOT_FOUND:
return H_NOT_FOUND;
case REMOVE_PARM:
return H_PARAMETER;
case REMOVE_HW:
return H_HARDWARE;
}
g_assert_not_reached();
}
#define H_BULK_REMOVE_TYPE 0xc000000000000000ULL
#define H_BULK_REMOVE_REQUEST 0x4000000000000000ULL
#define H_BULK_REMOVE_RESPONSE 0x8000000000000000ULL
#define H_BULK_REMOVE_END 0xc000000000000000ULL
#define H_BULK_REMOVE_CODE 0x3000000000000000ULL
#define H_BULK_REMOVE_SUCCESS 0x0000000000000000ULL
#define H_BULK_REMOVE_NOT_FOUND 0x1000000000000000ULL
#define H_BULK_REMOVE_PARM 0x2000000000000000ULL
#define H_BULK_REMOVE_HW 0x3000000000000000ULL
#define H_BULK_REMOVE_RC 0x0c00000000000000ULL
#define H_BULK_REMOVE_FLAGS 0x0300000000000000ULL
#define H_BULK_REMOVE_ABSOLUTE 0x0000000000000000ULL
#define H_BULK_REMOVE_ANDCOND 0x0100000000000000ULL
#define H_BULK_REMOVE_AVPN 0x0200000000000000ULL
#define H_BULK_REMOVE_PTEX 0x00ffffffffffffffULL
#define H_BULK_REMOVE_MAX_BATCH 4
static target_ulong h_bulk_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
target_ulong opcode, target_ulong *args)
{
CPUPPCState *env = &cpu->env;
int i;
target_ulong rc = H_SUCCESS;
for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
target_ulong *tsh = &args[i*2];
target_ulong tsl = args[i*2 + 1];
target_ulong v, r, ret;
if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
break;
} else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
return H_PARAMETER;
}
*tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
*tsh |= H_BULK_REMOVE_RESPONSE;
if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
*tsh |= H_BULK_REMOVE_PARM;
return H_PARAMETER;
}
ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
(*tsh & H_BULK_REMOVE_FLAGS) >> 26,
&v, &r);
*tsh |= ret << 60;
switch (ret) {
case REMOVE_SUCCESS:
*tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
break;
case REMOVE_PARM:
rc = H_PARAMETER;
goto exit;
case REMOVE_HW:
rc = H_HARDWARE;
goto exit;
}
}
exit:
check_tlb_flush(env, true);
return rc;
}
static target_ulong h_protect(PowerPCCPU *cpu, SpaprMachineState *spapr,
target_ulong opcode, target_ulong *args)
{
CPUPPCState *env = &cpu->env;
target_ulong flags = args[0];
target_ulong ptex = args[1];
target_ulong avpn = args[2];
const ppc_hash_pte64_t *hptes;
target_ulong v, r;
if (!valid_ptex(cpu, ptex)) {
return H_PARAMETER;
}
hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
v = ppc_hash64_hpte0(cpu, hptes, 0);
r = ppc_hash64_hpte1(cpu, hptes, 0);
ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
if ((v & HPTE64_V_VALID) == 0 ||
((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
return H_NOT_FOUND;
}
r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
r |= (flags << 55) & HPTE64_R_PP0;
r |= (flags << 48) & HPTE64_R_KEY_HI;
r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
spapr_store_hpte(cpu, ptex,
(v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
/* Flush the tlb */
check_tlb_flush(env, true);
/* Don't need a memory barrier, due to qemu's global lock */
spapr_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
return H_SUCCESS;
}
static target_ulong h_read(PowerPCCPU *cpu, SpaprMachineState *spapr,
target_ulong opcode, target_ulong *args)
{
target_ulong flags = args[0];
target_ulong ptex = args[1];
int i, ridx, n_entries = 1;
const ppc_hash_pte64_t *hptes;
if (!valid_ptex(cpu, ptex)) {
return H_PARAMETER;
}
if (flags & H_READ_4) {
/* Clear the two low order bits */
ptex &= ~(3ULL);
n_entries = 4;
}
hptes = ppc_hash64_map_hptes(cpu, ptex, n_entries);
for (i = 0, ridx = 0; i < n_entries; i++) {
args[ridx++] = ppc_hash64_hpte0(cpu, hptes, i);
args[ridx++] = ppc_hash64_hpte1(cpu, hptes, i);
}
ppc_hash64_unmap_hptes(cpu, hptes, ptex, n_entries);
return H_SUCCESS;
}
struct SpaprPendingHpt {
/* These fields are read-only after initialization */
int shift;
QemuThread thread;
/* These fields are protected by the BQL */
bool complete;
/* These fields are private to the preparation thread if
* !complete, otherwise protected by the BQL */
int ret;
void *hpt;
};
static void free_pending_hpt(SpaprPendingHpt *pending)
{
if (pending->hpt) {
qemu_vfree(pending->hpt);
}
g_free(pending);
}
static void *hpt_prepare_thread(void *opaque)
{
SpaprPendingHpt *pending = opaque;
size_t size = 1ULL << pending->shift;
pending->hpt = qemu_try_memalign(size, size);
if (pending->hpt) {
memset(pending->hpt, 0, size);
pending->ret = H_SUCCESS;
} else {
pending->ret = H_NO_MEM;
}
qemu_mutex_lock_iothread();
if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
/* Ready to go */
pending->complete = true;
} else {
/* We've been cancelled, clean ourselves up */
free_pending_hpt(pending);
}
qemu_mutex_unlock_iothread();
return NULL;
}
/* Must be called with BQL held */
static void cancel_hpt_prepare(SpaprMachineState *spapr)
{
SpaprPendingHpt *pending = spapr->pending_hpt;
/* Let the thread know it's cancelled */
spapr->pending_hpt = NULL;
if (!pending) {
/* Nothing to do */
return;
}
if (!pending->complete) {
/* thread will clean itself up */
return;
}
free_pending_hpt(pending);
}
target_ulong softmmu_resize_hpt_prepare(PowerPCCPU *cpu,
SpaprMachineState *spapr,
target_ulong shift)
{
SpaprPendingHpt *pending = spapr->pending_hpt;
if (pending) {
/* something already in progress */
if (pending->shift == shift) {
/* and it's suitable */
if (pending->complete) {
return pending->ret;
} else {
return H_LONG_BUSY_ORDER_100_MSEC;
}
}
/* not suitable, cancel and replace */
cancel_hpt_prepare(spapr);
}
if (!shift) {
/* nothing to do */
return H_SUCCESS;
}
/* start new prepare */
pending = g_new0(SpaprPendingHpt, 1);
pending->shift = shift;
pending->ret = H_HARDWARE;
qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
spapr->pending_hpt = pending;
/* In theory we could estimate the time more accurately based on
* the new size, but there's not much point */
return H_LONG_BUSY_ORDER_100_MSEC;
}
static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
{
uint8_t *addr = htab;
addr += pteg * HASH_PTEG_SIZE_64;
addr += slot * HASH_PTE_SIZE_64;
return ldq_p(addr);
}
static void new_hpte_store(void *htab, uint64_t pteg, int slot,
uint64_t pte0, uint64_t pte1)
{
uint8_t *addr = htab;
addr += pteg * HASH_PTEG_SIZE_64;
addr += slot * HASH_PTE_SIZE_64;
stq_p(addr, pte0);
stq_p(addr + HPTE64_DW1, pte1);
}
static int rehash_hpte(PowerPCCPU *cpu,
const ppc_hash_pte64_t *hptes,
void *old_hpt, uint64_t oldsize,
void *new_hpt, uint64_t newsize,
uint64_t pteg, int slot)
{
uint64_t old_hash_mask = (oldsize >> 7) - 1;
uint64_t new_hash_mask = (newsize >> 7) - 1;
target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
target_ulong pte1;
uint64_t avpn;
unsigned base_pg_shift;
uint64_t hash, new_pteg, replace_pte0;
if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
return H_SUCCESS;
}
pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
assert(base_pg_shift); /* H_ENTER shouldn't allow a bad encoding */
avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
if (pte0 & HPTE64_V_SECONDARY) {
pteg = ~pteg;
}
if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
uint64_t offset, vsid;
/* We only have 28 - 23 bits of offset in avpn */
offset = (avpn & 0x1f) << 23;
vsid = avpn >> 5;
/* We can find more bits from the pteg value */
if (base_pg_shift < 23) {
offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
}
hash = vsid ^ (offset >> base_pg_shift);
} else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
uint64_t offset, vsid;
/* We only have 40 - 23 bits of seg_off in avpn */
offset = (avpn & 0x1ffff) << 23;
vsid = avpn >> 17;
if (base_pg_shift < 23) {
offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
<< base_pg_shift;
}
hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
} else {
error_report("rehash_pte: Bad segment size in HPTE");
return H_HARDWARE;
}
new_pteg = hash & new_hash_mask;
if (pte0 & HPTE64_V_SECONDARY) {
assert(~pteg == (hash & old_hash_mask));
new_pteg = ~new_pteg;
} else {
assert(pteg == (hash & old_hash_mask));
}
assert((oldsize != newsize) || (pteg == new_pteg));
replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
/*
* Strictly speaking, we don't need all these tests, since we only
* ever rehash bolted HPTEs. We might in future handle non-bolted
* HPTEs, though so make the logic correct for those cases as
* well.
*/
if (replace_pte0 & HPTE64_V_VALID) {
assert(newsize < oldsize);
if (replace_pte0 & HPTE64_V_BOLTED) {
if (pte0 & HPTE64_V_BOLTED) {
/* Bolted collision, nothing we can do */
return H_PTEG_FULL;
} else {
/* Discard this hpte */
return H_SUCCESS;
}
}
}
new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
return H_SUCCESS;
}
static int rehash_hpt(PowerPCCPU *cpu,
void *old_hpt, uint64_t oldsize,
void *new_hpt, uint64_t newsize)
{
uint64_t n_ptegs = oldsize >> 7;
uint64_t pteg;
int slot;
int rc;
for (pteg = 0; pteg < n_ptegs; pteg++) {
hwaddr ptex = pteg * HPTES_PER_GROUP;
const ppc_hash_pte64_t *hptes
= ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
if (!hptes) {
return H_HARDWARE;
}
for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
pteg, slot);
if (rc != H_SUCCESS) {
ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
return rc;
}
}
ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
}
return H_SUCCESS;
}
target_ulong softmmu_resize_hpt_commit(PowerPCCPU *cpu,
SpaprMachineState *spapr,
target_ulong flags,
target_ulong shift)
{
SpaprPendingHpt *pending = spapr->pending_hpt;
int rc;
size_t newsize;
if (flags != 0) {
return H_PARAMETER;
}
if (!pending || (pending->shift != shift)) {
/* no matching prepare */
return H_CLOSED;
}
if (!pending->complete) {
/* prepare has not completed */
return H_BUSY;
}
/* Shouldn't have got past PREPARE without an HPT */
g_assert(spapr->htab_shift);
newsize = 1ULL << pending->shift;
rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
pending->hpt, newsize);
if (rc == H_SUCCESS) {
qemu_vfree(spapr->htab);
spapr->htab = pending->hpt;
spapr->htab_shift = pending->shift;
push_sregs_to_kvm_pr(spapr);
pending->hpt = NULL; /* so it's not free()d */
}
/* Clean up */
spapr->pending_hpt = NULL;
free_pending_hpt(pending);
return rc;
}
static void hypercall_register_types(void)
{
/* hcall-pft */
spapr_register_hypercall(H_ENTER, h_enter);
spapr_register_hypercall(H_REMOVE, h_remove);
spapr_register_hypercall(H_PROTECT, h_protect);
spapr_register_hypercall(H_READ, h_read);
/* hcall-bulk */
spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
}
type_init(hypercall_register_types)
