summaryrefslogtreecommitdiffstats
path: root/target
diff options
context:
space:
mode:
Diffstat (limited to 'target')
-rw-r--r--target/arm/cpregs.h5
-rw-r--r--target/arm/cpu.c32
-rw-r--r--target/arm/cpu.h103
-rw-r--r--target/arm/cpu64.c205
-rw-r--r--target/arm/helper-sme.h21
-rw-r--r--target/arm/helper.c213
-rw-r--r--target/arm/helper.h1
-rw-r--r--target/arm/internals.h4
-rw-r--r--target/arm/kvm64.c2
-rw-r--r--target/arm/machine.c34
-rw-r--r--target/arm/meson.build1
-rw-r--r--target/arm/ptw.c26
-rw-r--r--target/arm/sme_helper.c61
-rw-r--r--target/arm/syndrome.h14
-rw-r--r--target/arm/translate-a64.c46
-rw-r--r--target/arm/translate-a64.h38
-rw-r--r--target/arm/translate-sve.c36
-rw-r--r--target/arm/translate.h6
18 files changed, 714 insertions, 134 deletions
diff --git a/target/arm/cpregs.h b/target/arm/cpregs.h
index d9b678c2f1..d30758ee71 100644
--- a/target/arm/cpregs.h
+++ b/target/arm/cpregs.h
@@ -113,6 +113,11 @@ enum {
ARM_CP_EL3_NO_EL2_UNDEF = 1 << 16,
ARM_CP_EL3_NO_EL2_KEEP = 1 << 17,
ARM_CP_EL3_NO_EL2_C_NZ = 1 << 18,
+ /*
+ * Flag: Access check for this sysreg is constrained by the
+ * ARM pseudocode function CheckSMEAccess().
+ */
+ ARM_CP_SME = 1 << 19,
};
/*
diff --git a/target/arm/cpu.c b/target/arm/cpu.c
index 1b5d535788..bb44ad45aa 100644
--- a/target/arm/cpu.c
+++ b/target/arm/cpu.c
@@ -39,6 +39,7 @@
#include "hw/boards.h"
#endif
#include "sysemu/tcg.h"
+#include "sysemu/qtest.h"
#include "sysemu/hw_accel.h"
#include "kvm_arm.h"
#include "disas/capstone.h"
@@ -1122,11 +1123,13 @@ static void arm_cpu_initfn(Object *obj)
#ifdef CONFIG_USER_ONLY
# ifdef TARGET_AARCH64
/*
- * The linux kernel defaults to 512-bit vectors, when sve is supported.
- * See documentation for /proc/sys/abi/sve_default_vector_length, and
- * our corresponding sve-default-vector-length cpu property.
+ * The linux kernel defaults to 512-bit for SVE, and 256-bit for SME.
+ * These values were chosen to fit within the default signal frame.
+ * See documentation for /proc/sys/abi/{sve,sme}_default_vector_length,
+ * and our corresponding cpu property.
*/
cpu->sve_default_vq = 4;
+ cpu->sme_default_vq = 2;
# endif
#else
/* Our inbound IRQ and FIQ lines */
@@ -1421,6 +1424,7 @@ void arm_cpu_finalize_features(ARMCPU *cpu, Error **errp)
{
Error *local_err = NULL;
+#ifdef TARGET_AARCH64
if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
arm_cpu_sve_finalize(cpu, &local_err);
if (local_err != NULL) {
@@ -1428,6 +1432,12 @@ void arm_cpu_finalize_features(ARMCPU *cpu, Error **errp)
return;
}
+ arm_cpu_sme_finalize(cpu, &local_err);
+ if (local_err != NULL) {
+ error_propagate(errp, local_err);
+ return;
+ }
+
arm_cpu_pauth_finalize(cpu, &local_err);
if (local_err != NULL) {
error_propagate(errp, local_err);
@@ -1440,6 +1450,7 @@ void arm_cpu_finalize_features(ARMCPU *cpu, Error **errp)
return;
}
}
+#endif
if (kvm_enabled()) {
kvm_arm_steal_time_finalize(cpu, &local_err);
@@ -1490,25 +1501,32 @@ static void arm_cpu_realizefn(DeviceState *dev, Error **errp)
}
}
- if (kvm_enabled()) {
+ if (!tcg_enabled() && !qtest_enabled()) {
/*
+ * We assume that no accelerator except TCG (and the "not really an
+ * accelerator" qtest) can handle these features, because Arm hardware
+ * virtualization can't virtualize them.
+ *
* Catch all the cases which might cause us to create more than one
* address space for the CPU (otherwise we will assert() later in
* cpu_address_space_init()).
*/
if (arm_feature(env, ARM_FEATURE_M)) {
error_setg(errp,
- "Cannot enable KVM when using an M-profile guest CPU");
+ "Cannot enable %s when using an M-profile guest CPU",
+ current_accel_name());
return;
}
if (cpu->has_el3) {
error_setg(errp,
- "Cannot enable KVM when guest CPU has EL3 enabled");
+ "Cannot enable %s when guest CPU has EL3 enabled",
+ current_accel_name());
return;
}
if (cpu->tag_memory) {
error_setg(errp,
- "Cannot enable KVM when guest CPUs has MTE enabled");
+ "Cannot enable %s when guest CPUs has MTE enabled",
+ current_accel_name());
return;
}
}
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index df677b2d5d..4a4342f262 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -205,14 +205,8 @@ typedef struct {
#ifdef TARGET_AARCH64
# define ARM_MAX_VQ 16
-void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp);
-void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp);
-void arm_cpu_lpa2_finalize(ARMCPU *cpu, Error **errp);
#else
# define ARM_MAX_VQ 1
-static inline void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) { }
-static inline void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp) { }
-static inline void arm_cpu_lpa2_finalize(ARMCPU *cpu, Error **errp) { }
#endif
typedef struct ARMVectorReg {
@@ -258,6 +252,7 @@ typedef struct CPUArchState {
* nRW (also known as M[4]) is kept, inverted, in env->aarch64
* DAIF (exception masks) are kept in env->daif
* BTYPE is kept in env->btype
+ * SM and ZA are kept in env->svcr
* all other bits are stored in their correct places in env->pstate
*/
uint32_t pstate;
@@ -292,6 +287,7 @@ typedef struct CPUArchState {
uint32_t condexec_bits; /* IT bits. cpsr[15:10,26:25]. */
uint32_t btype; /* BTI branch type. spsr[11:10]. */
uint64_t daif; /* exception masks, in the bits they are in PSTATE */
+ uint64_t svcr; /* PSTATE.{SM,ZA} in the bits they are in SVCR */
uint64_t elr_el[4]; /* AArch64 exception link regs */
uint64_t sp_el[4]; /* AArch64 banked stack pointers */
@@ -474,6 +470,7 @@ typedef struct CPUArchState {
};
uint64_t tpidr_el[4];
};
+ uint64_t tpidr2_el0;
/* The secure banks of these registers don't map anywhere */
uint64_t tpidrurw_s;
uint64_t tpidrprw_s;
@@ -666,8 +663,8 @@ typedef struct CPUArchState {
float_status standard_fp_status;
float_status standard_fp_status_f16;
- /* ZCR_EL[1-3] */
- uint64_t zcr_el[4];
+ uint64_t zcr_el[4]; /* ZCR_EL[1-3] */
+ uint64_t smcr_el[4]; /* SMCR_EL[1-3] */
} vfp;
uint64_t exclusive_addr;
uint64_t exclusive_val;
@@ -691,6 +688,28 @@ typedef struct CPUArchState {
} keys;
uint64_t scxtnum_el[4];
+
+ /*
+ * SME ZA storage -- 256 x 256 byte array, with bytes in host word order,
+ * as we do with vfp.zregs[]. This corresponds to the architectural ZA
+ * array, where ZA[N] is in the least-significant bytes of env->zarray[N].
+ * When SVL is less than the architectural maximum, the accessible
+ * storage is restricted, such that if the SVL is X bytes the guest can
+ * see only the bottom X elements of zarray[], and only the least
+ * significant X bytes of each element of the array. (In other words,
+ * the observable part is always square.)
+ *
+ * The ZA storage can also be considered as a set of square tiles of
+ * elements of different sizes. The mapping from tiles to the ZA array
+ * is architecturally defined, such that for tiles of elements of esz
+ * bytes, the Nth row (or "horizontal slice") of tile T is in
+ * ZA[T + N * esz]. Note that this means that each tile is not contiguous
+ * in the ZA storage, because its rows are striped through the ZA array.
+ *
+ * Because this is so large, keep this toward the end of the reset area,
+ * to keep the offsets into the rest of the structure smaller.
+ */
+ ARMVectorReg zarray[ARM_MAX_VQ * 16];
#endif
#if defined(CONFIG_USER_ONLY)
@@ -782,6 +801,19 @@ typedef enum ARMPSCIState {
typedef struct ARMISARegisters ARMISARegisters;
+/*
+ * In map, each set bit is a supported vector length of (bit-number + 1) * 16
+ * bytes, i.e. each bit number + 1 is the vector length in quadwords.
+ *
+ * While processing properties during initialization, corresponding init bits
+ * are set for bits in sve_vq_map that have been set by properties.
+ *
+ * Bits set in supported represent valid vector lengths for the CPU type.
+ */
+typedef struct {
+ uint32_t map, init, supported;
+} ARMVQMap;
+
/**
* ARMCPU:
* @env: #CPUARMState
@@ -1028,23 +1060,11 @@ struct ArchCPU {
#ifdef CONFIG_USER_ONLY
/* Used to set the default vector length at process start. */
uint32_t sve_default_vq;
+ uint32_t sme_default_vq;
#endif
- /*
- * In sve_vq_map each set bit is a supported vector length of
- * (bit-number + 1) * 16 bytes, i.e. each bit number + 1 is the vector
- * length in quadwords.
- *
- * While processing properties during initialization, corresponding
- * sve_vq_init bits are set for bits in sve_vq_map that have been
- * set by properties.
- *
- * Bits set in sve_vq_supported represent valid vector lengths for
- * the CPU type.
- */
- uint32_t sve_vq_map;
- uint32_t sve_vq_init;
- uint32_t sve_vq_supported;
+ ARMVQMap sve_vq;
+ ARMVQMap sme_vq;
/* Generic timer counter frequency, in Hz */
uint64_t gt_cntfrq_hz;
@@ -1093,8 +1113,7 @@ int aarch64_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg);
void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq);
void aarch64_sve_change_el(CPUARMState *env, int old_el,
int new_el, bool el0_a64);
-void aarch64_add_sve_properties(Object *obj);
-void aarch64_add_pauth_properties(Object *obj);
+void arm_reset_sve_state(CPUARMState *env);
/*
* SVE registers are encoded in KVM's memory in an endianness-invariant format.
@@ -1125,7 +1144,6 @@ static inline void aarch64_sve_narrow_vq(CPUARMState *env, unsigned vq) { }
static inline void aarch64_sve_change_el(CPUARMState *env, int o,
int n, bool a)
{ }
-static inline void aarch64_add_sve_properties(Object *obj) { }
#endif
void aarch64_sync_32_to_64(CPUARMState *env);
@@ -1133,15 +1151,21 @@ void aarch64_sync_64_to_32(CPUARMState *env);
int fp_exception_el(CPUARMState *env, int cur_el);
int sve_exception_el(CPUARMState *env, int cur_el);
+int sme_exception_el(CPUARMState *env, int cur_el);
/**
- * sve_vqm1_for_el:
+ * sve_vqm1_for_el_sm:
* @env: CPUARMState
* @el: exception level
+ * @sm: streaming mode
*
- * Compute the current SVE vector length for @el, in units of
+ * Compute the current vector length for @el & @sm, in units of
* Quadwords Minus 1 -- the same scale used for ZCR_ELx.LEN.
+ * If @sm, compute for SVL, otherwise NVL.
*/
+uint32_t sve_vqm1_for_el_sm(CPUARMState *env, int el, bool sm);
+
+/* Likewise, but using @sm = PSTATE.SM. */
uint32_t sve_vqm1_for_el(CPUARMState *env, int el);
static inline bool is_a64(CPUARMState *env)
@@ -1426,6 +1450,14 @@ FIELD(CPTR_EL3, TCPAC, 31, 1)
#define PSTATE_MODE_EL1t 4
#define PSTATE_MODE_EL0t 0
+/* PSTATE bits that are accessed via SVCR and not stored in SPSR_ELx. */
+FIELD(SVCR, SM, 0, 1)
+FIELD(SVCR, ZA, 1, 1)
+
+/* Fields for SMCR_ELx. */
+FIELD(SMCR, LEN, 0, 4)
+FIELD(SMCR, FA64, 31, 1)
+
/* Write a new value to v7m.exception, thus transitioning into or out
* of Handler mode; this may result in a change of active stack pointer.
*/
@@ -3147,6 +3179,10 @@ FIELD(TBFLAG_A64, ATA, 15, 1)
FIELD(TBFLAG_A64, TCMA, 16, 2)
FIELD(TBFLAG_A64, MTE_ACTIVE, 18, 1)
FIELD(TBFLAG_A64, MTE0_ACTIVE, 19, 1)
+FIELD(TBFLAG_A64, SMEEXC_EL, 20, 2)
+FIELD(TBFLAG_A64, PSTATE_SM, 22, 1)
+FIELD(TBFLAG_A64, PSTATE_ZA, 23, 1)
+FIELD(TBFLAG_A64, SVL, 24, 4)
/*
* Helpers for using the above.
@@ -3192,6 +3228,17 @@ static inline int sve_vq(CPUARMState *env)
return EX_TBFLAG_A64(env->hflags, VL) + 1;
}
+/**
+ * sme_vq
+ * @env: the cpu context
+ *
+ * Return the SVL cached within env->hflags, in units of quadwords.
+ */
+static inline int sme_vq(CPUARMState *env)
+{
+ return EX_TBFLAG_A64(env->hflags, SVL) + 1;
+}
+
static inline bool bswap_code(bool sctlr_b)
{
#ifdef CONFIG_USER_ONLY
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 15665c962b..19188d6cc2 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -355,8 +355,8 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp)
* any of the above. Finally, if SVE is not disabled, then at least one
* vector length must be enabled.
*/
- uint32_t vq_map = cpu->sve_vq_map;
- uint32_t vq_init = cpu->sve_vq_init;
+ uint32_t vq_map = cpu->sve_vq.map;
+ uint32_t vq_init = cpu->sve_vq.init;
uint32_t vq_supported;
uint32_t vq_mask = 0;
uint32_t tmp, vq, max_vq = 0;
@@ -369,14 +369,14 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp)
*/
if (kvm_enabled()) {
if (kvm_arm_sve_supported()) {
- cpu->sve_vq_supported = kvm_arm_sve_get_vls(CPU(cpu));
- vq_supported = cpu->sve_vq_supported;
+ cpu->sve_vq.supported = kvm_arm_sve_get_vls(CPU(cpu));
+ vq_supported = cpu->sve_vq.supported;
} else {
assert(!cpu_isar_feature(aa64_sve, cpu));
vq_supported = 0;
}
} else {
- vq_supported = cpu->sve_vq_supported;
+ vq_supported = cpu->sve_vq.supported;
}
/*
@@ -487,8 +487,13 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp)
"using only sve<N> properties.\n");
} else {
error_setg(errp, "cannot enable sve%d", vq * 128);
- error_append_hint(errp, "This CPU does not support "
- "the vector length %d-bits.\n", vq * 128);
+ if (vq_supported) {
+ error_append_hint(errp, "This CPU does not support "
+ "the vector length %d-bits.\n", vq * 128);
+ } else {
+ error_append_hint(errp, "SVE not supported by KVM "
+ "on this host\n");
+ }
}
return;
} else {
@@ -529,7 +534,7 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp)
/* From now on sve_max_vq is the actual maximum supported length. */
cpu->sve_max_vq = max_vq;
- cpu->sve_vq_map = vq_map;
+ cpu->sve_vq.map = vq_map;
}
static void cpu_max_get_sve_max_vq(Object *obj, Visitor *v, const char *name,
@@ -574,31 +579,34 @@ static void cpu_max_set_sve_max_vq(Object *obj, Visitor *v, const char *name,
}
/*
- * Note that cpu_arm_get/set_sve_vq cannot use the simpler
- * object_property_add_bool interface because they make use
- * of the contents of "name" to determine which bit on which
- * to operate.
+ * Note that cpu_arm_{get,set}_vq cannot use the simpler
+ * object_property_add_bool interface because they make use of the
+ * contents of "name" to determine which bit on which to operate.
*/
-static void cpu_arm_get_sve_vq(Object *obj, Visitor *v, const char *name,
- void *opaque, Error **errp)
+static void cpu_arm_get_vq(Object *obj, Visitor *v, const char *name,
+ void *opaque, Error **errp)
{
ARMCPU *cpu = ARM_CPU(obj);
+ ARMVQMap *vq_map = opaque;
uint32_t vq = atoi(&name[3]) / 128;
+ bool sve = vq_map == &cpu->sve_vq;
bool value;
- /* All vector lengths are disabled when SVE is off. */
- if (!cpu_isar_feature(aa64_sve, cpu)) {
+ /* All vector lengths are disabled when feature is off. */
+ if (sve
+ ? !cpu_isar_feature(aa64_sve, cpu)
+ : !cpu_isar_feature(aa64_sme, cpu)) {
value = false;
} else {
- value = extract32(cpu->sve_vq_map, vq - 1, 1);
+ value = extract32(vq_map->map, vq - 1, 1);
}
visit_type_bool(v, name, &value, errp);
}
-static void cpu_arm_set_sve_vq(Object *obj, Visitor *v, const char *name,
- void *opaque, Error **errp)
+static void cpu_arm_set_vq(Object *obj, Visitor *v, const char *name,
+ void *opaque, Error **errp)
{
- ARMCPU *cpu = ARM_CPU(obj);
+ ARMVQMap *vq_map = opaque;
uint32_t vq = atoi(&name[3]) / 128;
bool value;
@@ -606,14 +614,8 @@ static void cpu_arm_set_sve_vq(Object *obj, Visitor *v, const char *name,
return;
}
- if (value && kvm_enabled() && !kvm_arm_sve_supported()) {
- error_setg(errp, "cannot enable %s", name);
- error_append_hint(errp, "SVE not supported by KVM on this host\n");
- return;
- }
-
- cpu->sve_vq_map = deposit32(cpu->sve_vq_map, vq - 1, 1, value);
- cpu->sve_vq_init |= 1 << (vq - 1);
+ vq_map->map = deposit32(vq_map->map, vq - 1, 1, value);
+ vq_map->init |= 1 << (vq - 1);
}
static bool cpu_arm_get_sve(Object *obj, Error **errp)
@@ -637,13 +639,85 @@ static void cpu_arm_set_sve(Object *obj, bool value, Error **errp)
cpu->isar.id_aa64pfr0 = t;
}
-#ifdef CONFIG_USER_ONLY
-/* Mirror linux /proc/sys/abi/sve_default_vector_length. */
-static void cpu_arm_set_sve_default_vec_len(Object *obj, Visitor *v,
- const char *name, void *opaque,
- Error **errp)
+void arm_cpu_sme_finalize(ARMCPU *cpu, Error **errp)
+{
+ uint32_t vq_map = cpu->sme_vq.map;
+ uint32_t vq_init = cpu->sme_vq.init;
+ uint32_t vq_supported = cpu->sme_vq.supported;
+ uint32_t vq;
+
+ if (vq_map == 0) {
+ if (!cpu_isar_feature(aa64_sme, cpu)) {
+ cpu->isar.id_aa64smfr0 = 0;
+ return;
+ }
+
+ /* TODO: KVM will require limitations via SMCR_EL2. */
+ vq_map = vq_supported & ~vq_init;
+
+ if (vq_map == 0) {
+ vq = ctz32(vq_supported) + 1;
+ error_setg(errp, "cannot disable sme%d", vq * 128);
+ error_append_hint(errp, "All SME vector lengths are disabled.\n");
+ error_append_hint(errp, "With SME enabled, at least one "
+ "vector length must be enabled.\n");
+ return;
+ }
+ } else {
+ if (!cpu_isar_feature(aa64_sme, cpu)) {
+ vq = 32 - clz32(vq_map);
+ error_setg(errp, "cannot enable sme%d", vq * 128);
+ error_append_hint(errp, "SME must be enabled to enable "
+ "vector lengths.\n");
+ error_append_hint(errp, "Add sme=on to the CPU property list.\n");
+ return;
+ }
+ /* TODO: KVM will require limitations via SMCR_EL2. */
+ }
+
+ cpu->sme_vq.map = vq_map;
+}
+
+static bool cpu_arm_get_sme(Object *obj, Error **errp)
+{
+ ARMCPU *cpu = ARM_CPU(obj);
+ return cpu_isar_feature(aa64_sme, cpu);
+}
+
+static void cpu_arm_set_sme(Object *obj, bool value, Error **errp)
{
ARMCPU *cpu = ARM_CPU(obj);
+ uint64_t t;
+
+ t = cpu->isar.id_aa64pfr1;
+ t = FIELD_DP64(t, ID_AA64PFR1, SME, value);
+ cpu->isar.id_aa64pfr1 = t;
+}
+
+static bool cpu_arm_get_sme_fa64(Object *obj, Error **errp)
+{
+ ARMCPU *cpu = ARM_CPU(obj);
+ return cpu_isar_feature(aa64_sme, cpu) &&
+ cpu_isar_feature(aa64_sme_fa64, cpu);
+}
+
+static void cpu_arm_set_sme_fa64(Object *obj, bool value, Error **errp)
+{
+ ARMCPU *cpu = ARM_CPU(obj);
+ uint64_t t;
+
+ t = cpu->isar.id_aa64smfr0;
+ t = FIELD_DP64(t, ID_AA64SMFR0, FA64, value);
+ cpu->isar.id_aa64smfr0 = t;
+}
+
+#ifdef CONFIG_USER_ONLY
+/* Mirror linux /proc/sys/abi/{sve,sme}_default_vector_length. */
+static void cpu_arm_set_default_vec_len(Object *obj, Visitor *v,
+ const char *name, void *opaque,
+ Error **errp)
+{
+ uint32_t *ptr_default_vq = opaque;
int32_t default_len, default_vq, remainder;
if (!visit_type_int32(v, name, &default_len, errp)) {
@@ -652,7 +726,7 @@ static void cpu_arm_set_sve_default_vec_len(Object *obj, Visitor *v,
/* Undocumented, but the kernel allows -1 to indicate "maximum". */
if (default_len == -1) {
- cpu->sve_default_vq = ARM_MAX_VQ;
+ *ptr_default_vq = ARM_MAX_VQ;
return;
}
@@ -664,7 +738,11 @@ static void cpu_arm_set_sve_default_vec_len(Object *obj, Visitor *v,
* and is the maximum architectural width of ZCR_ELx.LEN.
*/
if (remainder || default_vq < 1 || default_vq > 512) {
- error_setg(errp, "cannot set sve-default-vector-length");
+ ARMCPU *cpu = ARM_CPU(obj);
+ const char *which =
+ (ptr_default_vq == &cpu->sve_default_vq ? "sve" : "sme");
+
+ error_setg(errp, "cannot set %s-default-vector-length", which);
if (remainder) {
error_append_hint(errp, "Vector length not a multiple of 16\n");
} else if (default_vq < 1) {
@@ -676,22 +754,23 @@ static void cpu_arm_set_sve_default_vec_len(Object *obj, Visitor *v,
return;
}
- cpu->sve_default_vq = default_vq;
+ *ptr_default_vq = default_vq;
}
-static void cpu_arm_get_sve_default_vec_len(Object *obj, Visitor *v,
- const char *name, void *opaque,
- Error **errp)
+static void cpu_arm_get_default_vec_len(Object *obj, Visitor *v,
+ const char *name, void *opaque,
+ Error **errp)
{
- ARMCPU *cpu = ARM_CPU(obj);
- int32_t value = cpu->sve_default_vq * 16;
+ uint32_t *ptr_default_vq = opaque;
+ int32_t value = *ptr_default_vq * 16;
visit_type_int32(v, name, &value, errp);
}
#endif
-void aarch64_add_sve_properties(Object *obj)
+static void aarch64_add_sve_properties(Object *obj)
{
+ ARMCPU *cpu = ARM_CPU(obj);
uint32_t vq;
object_property_add_bool(obj, "sve", cpu_arm_get_sve, cpu_arm_set_sve);
@@ -699,15 +778,41 @@ void aarch64_add_sve_properties(Object *obj)
for (vq = 1; vq <= ARM_MAX_VQ; ++vq) {
char name[8];
sprintf(name, "sve%d", vq * 128);
- object_property_add(obj, name, "bool", cpu_arm_get_sve_vq,
- cpu_arm_set_sve_vq, NULL, NULL);
+ object_property_add(obj, name, "bool", cpu_arm_get_vq,
+ cpu_arm_set_vq, NULL, &cpu->sve_vq);
}
#ifdef CONFIG_USER_ONLY
/* Mirror linux /proc/sys/abi/sve_default_vector_length. */
object_property_add(obj, "sve-default-vector-length", "int32",
- cpu_arm_get_sve_default_vec_len,
- cpu_arm_set_sve_default_vec_len, NULL, NULL);
+ cpu_arm_get_default_vec_len,
+ cpu_arm_set_default_vec_len, NULL,
+ &cpu->sve_default_vq);
+#endif
+}
+
+static void aarch64_add_sme_properties(Object *obj)
+{
+ ARMCPU *cpu = ARM_CPU(obj);
+ uint32_t vq;
+
+ object_property_add_bool(obj, "sme", cpu_arm_get_sme, cpu_arm_set_sme);
+ object_property_add_bool(obj, "sme_fa64", cpu_arm_get_sme_fa64,
+ cpu_arm_set_sme_fa64);
+
+ for (vq = 1; vq <= ARM_MAX_VQ; vq <<= 1) {
+ char name[8];
+ sprintf(name, "sme%d", vq * 128);
+ object_property_add(obj, name, "bool", cpu_arm_get_vq,
+ cpu_arm_set_vq, NULL, &cpu->sme_vq);
+ }
+
+#ifdef CONFIG_USER_ONLY
+ /* Mirror linux /proc/sys/abi/sme_default_vector_length. */
+ object_property_add(obj, "sme-default-vector-length", "int32",
+ cpu_arm_get_default_vec_len,
+ cpu_arm_set_default_vec_len, NULL,
+ &cpu->sme_default_vq);
#endif
}
@@ -751,7 +856,7 @@ static Property arm_cpu_pauth_property =
static Property arm_cpu_pauth_impdef_property =
DEFINE_PROP_BOOL("pauth-impdef", ARMCPU, prop_pauth_impdef, false);
-void aarch64_add_pauth_properties(Object *obj)
+static void aarch64_add_pauth_properties(Object *obj)
{
ARMCPU *cpu = ARM_CPU(obj);
@@ -975,10 +1080,12 @@ static void aarch64_max_initfn(Object *obj)
cpu->dcz_blocksize = 7; /* 512 bytes */
#endif
- cpu->sve_vq_supported = MAKE_64BIT_MASK(0, ARM_MAX_VQ);
+ cpu->sve_vq.supported = MAKE_64BIT_MASK(0, ARM_MAX_VQ);
+ cpu->sme_vq.supported = SVE_VQ_POW2_MAP;
aarch64_add_pauth_properties(obj);
aarch64_add_sve_properties(obj);
+ aarch64_add_sme_properties(obj);
object_property_add(obj, "sve-max-vq", "uint32", cpu_max_get_sve_max_vq,
cpu_max_set_sve_max_vq, NULL, NULL);
qdev_property_add_static(DEVICE(obj), &arm_cpu_lpa2_property);
@@ -1024,7 +1131,7 @@ static void aarch64_a64fx_initfn(Object *obj)
/* The A64FX supports only 128, 256 and 512 bit vector lengths */
aarch64_add_sve_properties(obj);
- cpu->sve_vq_supported = (1 << 0) /* 128bit */
+ cpu->sve_vq.supported = (1 << 0) /* 128bit */
| (1 << 1) /* 256bit */
| (1 << 3); /* 512bit */
diff --git a/target/arm/helper-sme.h b/target/arm/helper-sme.h
new file mode 100644
index 0000000000..3bd48c235f
--- /dev/null
+++ b/target/arm/helper-sme.h
@@ -0,0 +1,21 @@
+/*
+ * AArch64 SME specific helper definitions
+ *
+ * Copyright (c) 2022 Linaro, Ltd
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+DEF_HELPER_FLAGS_2(set_pstate_sm, TCG_CALL_NO_RWG, void, env, i32)
+DEF_HELPER_FLAGS_2(set_pstate_za, TCG_CALL_NO_RWG, void, env, i32)
diff --git a/target/arm/helper.c b/target/arm/helper.c
index 6457e6301c..d2886a123a 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -5879,6 +5879,8 @@ static void define_arm_vh_e2h_redirects_aliases(ARMCPU *cpu)
*/
{ K(3, 0, 1, 2, 0), K(3, 4, 1, 2, 0), K(3, 5, 1, 2, 0),
"ZCR_EL1", "ZCR_EL2", "ZCR_EL12", isar_feature_aa64_sve },
+ { K(3, 0, 1, 2, 6), K(3, 4, 1, 2, 6), K(3, 5, 1, 2, 6),
+ "SMCR_EL1", "SMCR_EL2", "SMCR_EL12", isar_feature_aa64_sme },
{ K(3, 0, 5, 6, 0), K(3, 4, 5, 6, 0), K(3, 5, 5, 6, 0),
"TFSR_EL1", "TFSR_EL2", "TFSR_EL12", isar_feature_aa64_mte },
@@ -6219,25 +6221,92 @@ int sve_exception_el(CPUARMState *env, int el)
}
/*
+ * Return the exception level to which exceptions should be taken for SME.
+ * C.f. the ARM pseudocode function CheckSMEAccess.
+ */
+int sme_exception_el(CPUARMState *env, int el)
+{
+#ifndef CONFIG_USER_ONLY
+ if (el <= 1 && !el_is_in_host(env, el)) {
+ switch (FIELD_EX64(env->cp15.cpacr_el1, CPACR_EL1, SMEN)) {
+ case 1:
+ if (el != 0) {
+ break;
+ }
+ /* fall through */
+ case 0:
+ case 2:
+ return 1;
+ }
+ }
+
+ if (el <= 2 && arm_is_el2_enabled(env)) {
+ /* CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). */
+ if (env->cp15.hcr_el2 & HCR_E2H) {
+ switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, SMEN)) {
+ case 1:
+ if (el != 0 || !(env->cp15.hcr_el2 & HCR_TGE)) {
+ break;
+ }
+ /* fall through */
+ case 0:
+ case 2:
+ return 2;
+ }
+ } else {
+ if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TSM)) {
+ return 2;
+ }
+ }
+ }
+
+ /* CPTR_EL3. Since ESM is negative we must check for EL3. */
+ if (arm_feature(env, ARM_FEATURE_EL3)
+ && !FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, ESM)) {
+ return 3;
+ }
+#endif
+ return 0;
+}
+
+/*
* Given that SVE is enabled, return the vector length for EL.
*/
-uint32_t sve_vqm1_for_el(CPUARMState *env, int el)
+uint32_t sve_vqm1_for_el_sm(CPUARMState *env, int el, bool sm)
{
ARMCPU *cpu = env_archcpu(env);
- uint32_t len = cpu->sve_max_vq - 1;
+ uint64_t *cr = env->vfp.zcr_el;
+ uint32_t map = cpu->sve_vq.map;
+ uint32_t len = ARM_MAX_VQ - 1;
+
+ if (sm) {
+ cr = env->vfp.smcr_el;
+ map = cpu->sme_vq.map;
+ }
if (el <= 1 && !el_is_in_host(env, el)) {
- len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[1]);
+ len = MIN(len, 0xf & (uint32_t)cr[1]);
}
if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2)) {
- len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[2]);
+ len = MIN(len, 0xf & (uint32_t)cr[2]);
}
if (arm_feature(env, ARM_FEATURE_EL3)) {
- len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[3]);
+ len = MIN(len, 0xf & (uint32_t)cr[3]);
+ }
+
+ map &= MAKE_64BIT_MASK(0, len + 1);
+ if (map != 0) {
+ return 31 - clz32(map);
}
- len = 31 - clz32(cpu->sve_vq_map & MAKE_64BIT_MASK(0, len + 1));
- return len;
+ /* Bit 0 is always set for Normal SVE -- not so for Streaming SVE. */
+ assert(sm);
+ return ctz32(cpu->sme_vq.map);
+}
+
+uint32_t sve_vqm1_for_el(CPUARMState *env, int el)
+{
+ return sve_vqm1_for_el_sm(env, el, FIELD_EX64(env->svcr, SVCR, SM));
}
static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -6279,6 +6348,120 @@ static const ARMCPRegInfo zcr_reginfo[] = {
.writefn = zcr_write, .raw_writefn = raw_write },
};
+#ifdef TARGET_AARCH64
+static CPAccessResult access_tpidr2(CPUARMState *env, const ARMCPRegInfo *ri,
+ bool isread)
+{
+ int el = arm_current_el(env);
+
+ if (el == 0) {
+ uint64_t sctlr = arm_sctlr(env, el);
+ if (!(sctlr & SCTLR_EnTP2)) {
+ return CP_ACCESS_TRAP;
+ }
+ }
+ /* TODO: FEAT_FGT */
+ if (el < 3
+ && arm_feature(env, ARM_FEATURE_EL3)
+ && !(env->cp15.scr_el3 & SCR_ENTP2)) {
+ return CP_ACCESS_TRAP_EL3;
+ }
+ return CP_ACCESS_OK;
+}
+
+static CPAccessResult access_esm(CPUARMState *env, const ARMCPRegInfo *ri,
+ bool isread)
+{
+ /* TODO: FEAT_FGT for SMPRI_EL1 but not SMPRIMAP_EL2 */
+ if (arm_current_el(env) < 3
+ && arm_feature(env, ARM_FEATURE_EL3)
+ && !FIELD_EX64(env->cp15.cptr_el[3], CPTR_EL3, ESM)) {
+ return CP_ACCESS_TRAP_EL3;
+ }
+ return CP_ACCESS_OK;
+}
+
+static void svcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ helper_set_pstate_sm(env, FIELD_EX64(value, SVCR, SM));
+ helper_set_pstate_za(env, FIELD_EX64(value, SVCR, ZA));
+ arm_rebuild_hflags(env);
+}
+
+static void smcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ int cur_el = arm_current_el(env);
+ int old_len = sve_vqm1_for_el(env, cur_el);
+ int new_len;
+
+ QEMU_BUILD_BUG_ON(ARM_MAX_VQ > R_SMCR_LEN_MASK + 1);
+ value &= R_SMCR_LEN_MASK | R_SMCR_FA64_MASK;
+ raw_write(env, ri, value);
+
+ /*
+ * Note that it is CONSTRAINED UNPREDICTABLE what happens to ZA storage
+ * when SVL is widened (old values kept, or zeros). Choose to keep the
+ * current values for simplicity. But for QEMU internals, we must still
+ * apply the narrower SVL to the Zregs and Pregs -- see the comment
+ * above aarch64_sve_narrow_vq.
+ */
+ new_len = sve_vqm1_for_el(env, cur_el);
+ if (new_len < old_len) {
+ aarch64_sve_narrow_vq(env, new_len + 1);
+ }
+}
+
+static const ARMCPRegInfo sme_reginfo[] = {
+ { .name = "TPIDR2_EL0", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 3, .crn = 13, .crm = 0, .opc2 = 5,
+ .access = PL0_RW, .accessfn = access_tpidr2,
+ .fieldoffset = offsetof(CPUARMState, cp15.tpidr2_el0) },
+ { .name = "SVCR", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 3, .crn = 4, .crm = 2, .opc2 = 2,
+ .access = PL0_RW, .type = ARM_CP_SME,
+ .fieldoffset = offsetof(CPUARMState, svcr),
+ .writefn = svcr_write, .raw_writefn = raw_write },
+ { .name = "SMCR_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 6,
+ .access = PL1_RW, .type = ARM_CP_SME,
+ .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[1]),
+ .writefn = smcr_write, .raw_writefn = raw_write },
+ { .name = "SMCR_EL2", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 6,
+ .access = PL2_RW, .type = ARM_CP_SME,
+ .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[2]),
+ .writefn = smcr_write, .raw_writefn = raw_write },
+ { .name = "SMCR_EL3", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 2, .opc2 = 6,
+ .access = PL3_RW, .type = ARM_CP_SME,
+ .fieldoffset = offsetof(CPUARMState, vfp.smcr_el[3]),
+ .writefn = smcr_write, .raw_writefn = raw_write },
+ { .name = "SMIDR_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 6,
+ .access = PL1_R, .accessfn = access_aa64_tid1,
+ /*
+ * IMPLEMENTOR = 0 (software)
+ * REVISION = 0 (implementation defined)
+ * SMPS = 0 (no streaming execution priority in QEMU)
+ * AFFINITY = 0 (streaming sve mode not shared with other PEs)
+ */
+ .type = ARM_CP_CONST, .resetvalue = 0, },
+ /*
+ * Because SMIDR_EL1.SMPS is 0, SMPRI_EL1 and SMPRIMAP_EL2 are RES 0.
+ */
+ { .name = "SMPRI_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 4,
+ .access = PL1_RW, .accessfn = access_esm,
+ .type = ARM_CP_CONST, .resetvalue = 0 },
+ { .name = "SMPRIMAP_EL2", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 5,
+ .access = PL2_RW, .accessfn = access_esm,
+ .type = ARM_CP_CONST, .resetvalue = 0 },
+};
+#endif /* TARGET_AARCH64 */
+
void hw_watchpoint_update(ARMCPU *cpu, int n)
{
CPUARMState *env = &cpu->env;
@@ -8440,6 +8623,9 @@ void register_cp_regs_for_features(ARMCPU *cpu)
}
#ifdef TARGET_AARCH64
+ if (cpu_isar_feature(aa64_sme, cpu)) {
+ define_arm_cp_regs(cpu, sme_reginfo);
+ }
if (cpu_isar_feature(aa64_pauth, cpu)) {
define_arm_cp_regs(cpu, pauth_reginfo);
}
@@ -11165,6 +11351,19 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el,
}
DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el);
}
+ if (cpu_isar_feature(aa64_sme, env_archcpu(env))) {
+ int sme_el = sme_exception_el(env, el);
+
+ DP_TBFLAG_A64(flags, SMEEXC_EL, sme_el);
+ if (sme_el == 0) {
+ /* Similarly, do not compute SVL if SME is disabled. */
+ DP_TBFLAG_A64(flags, SVL, sve_vqm1_for_el_sm(env, el, true));
+ }
+ if (FIELD_EX64(env->svcr, SVCR, SM)) {
+ DP_TBFLAG_A64(flags, PSTATE_SM, 1);
+ }
+ DP_TBFLAG_A64(flags, PSTATE_ZA, FIELD_EX64(env->svcr, SVCR, ZA));
+ }
sctlr = regime_sctlr(env, stage1);
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 07d45faf49..3a8ce42ab0 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -1022,6 +1022,7 @@ DEF_HELPER_FLAGS_6(gvec_bfmlal_idx, TCG_CALL_NO_RWG,
#ifdef TARGET_AARCH64
#include "helper-a64.h"
#include "helper-sve.h"
+#include "helper-sme.h"
#endif
#include "helper-mve.h"
diff --git a/target/arm/internals.h b/target/arm/internals.h
index 6f94f3019d..c66f74a0db 100644
--- a/target/arm/internals.h
+++ b/target/arm/internals.h
@@ -1288,6 +1288,10 @@ int arm_gdb_get_svereg(CPUARMState *env, GByteArray *buf, int reg);
int arm_gdb_set_svereg(CPUARMState *env, uint8_t *buf, int reg);
int aarch64_fpu_gdb_get_reg(CPUARMState *env, GByteArray *buf, int reg);
int aarch64_fpu_gdb_set_reg(CPUARMState *env, uint8_t *buf, int reg);
+void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp);
+void arm_cpu_sme_finalize(ARMCPU *cpu, Error **errp);
+void arm_cpu_pauth_finalize(ARMCPU *cpu, Error **errp);
+void arm_cpu_lpa2_finalize(ARMCPU *cpu, Error **errp);
#endif
#ifdef CONFIG_USER_ONLY
diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c
index ff8f65da22..d16d4ea250 100644
--- a/target/arm/kvm64.c
+++ b/target/arm/kvm64.c
@@ -820,7 +820,7 @@ uint32_t kvm_arm_sve_get_vls(CPUState *cs)
static int kvm_arm_sve_set_vls(CPUState *cs)
{
ARMCPU *cpu = ARM_CPU(cs);
- uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq_map };
+ uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq.map };
struct kvm_one_reg reg = {
.id = KVM_REG_ARM64_SVE_VLS,
.addr = (uint64_t)&vls[0],
diff --git a/target/arm/machine.c b/target/arm/machine.c
index 285e387d2c..54c5c62433 100644
--- a/target/arm/machine.c
+++ b/target/arm/machine.c
@@ -167,6 +167,39 @@ static const VMStateDescription vmstate_sve = {
VMSTATE_END_OF_LIST()
}
};
+
+static const VMStateDescription vmstate_vreg = {
+ .name = "vreg",
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .fields = (VMStateField[]) {
+ VMSTATE_UINT64_ARRAY(d, ARMVectorReg, ARM_MAX_VQ * 2),
+ VMSTATE_END_OF_LIST()
+ }
+};
+
+static bool za_needed(void *opaque)
+{
+ ARMCPU *cpu = opaque;
+
+ /*
+ * When ZA storage is disabled, its contents are discarded.
+ * It will be zeroed when ZA storage is re-enabled.
+ */
+ return FIELD_EX64(cpu->env.svcr, SVCR, ZA);
+}
+
+static const VMStateDescription vmstate_za = {
+ .name = "cpu/sme",
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .needed = za_needed,
+ .fields = (VMStateField[]) {
+ VMSTATE_STRUCT_ARRAY(env.zarray, ARMCPU, ARM_MAX_VQ * 16, 0,
+ vmstate_vreg, ARMVectorReg),
+ VMSTATE_END_OF_LIST()
+ }
+};
#endif /* AARCH64 */
static bool serror_needed(void *opaque)
@@ -884,6 +917,7 @@ const VMStateDescription vmstate_arm_cpu = {
&vmstate_m_security,
#ifdef TARGET_AARCH64
&vmstate_sve,
+ &vmstate_za,
#endif
&vmstate_serror,
&vmstate_irq_line_state,
diff --git a/target/arm/meson.build b/target/arm/meson.build
index ac571fc45d..43dc600547 100644
--- a/target/arm/meson.build
+++ b/target/arm/meson.build
@@ -47,6 +47,7 @@ arm_ss.add(when: 'TARGET_AARCH64', if_true: files(
'mte_helper.c',
'pauth_helper.c',
'sve_helper.c',
+ 'sme_helper.c',
'translate-a64.c',
'translate-sve.c',
))
diff --git a/target/arm/ptw.c b/target/arm/ptw.c
index 4d97a24808..da478104f0 100644
--- a/target/arm/ptw.c
+++ b/target/arm/ptw.c
@@ -36,15 +36,29 @@ static const uint8_t pamax_map[] = {
/* The cpu-specific constant value of PAMax; also used by hw/arm/virt. */
unsigned int arm_pamax(ARMCPU *cpu)
{
- unsigned int parange =
- FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE);
+ if (arm_feature(&cpu->env, ARM_FEATURE_AARCH64)) {
+ unsigned int parange =
+ FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE);
+
+ /*
+ * id_aa64mmfr0 is a read-only register so values outside of the
+ * supported mappings can be considered an implementation error.
+ */
+ assert(parange < ARRAY_SIZE(pamax_map));
+ return pamax_map[parange];
+ }
/*
- * id_aa64mmfr0 is a read-only register so values outside of the
- * supported mappings can be considered an implementation error.
+ * In machvirt_init, we call arm_pamax on a cpu that is not fully
+ * initialized, so we can't rely on the propagation done in realize.
*/
- assert(parange < ARRAY_SIZE(pamax_map));
- return pamax_map[parange];
+ if (arm_feature(&cpu->env, ARM_FEATURE_LPAE) ||
+ arm_feature(&cpu->env, ARM_FEATURE_V7VE)) {
+ /* v7 with LPAE */
+ return 40;
+ }
+ /* Anything else */
+ return 32;
}
/*
diff --git a/target/arm/sme_helper.c b/target/arm/sme_helper.c
new file mode 100644
index 0000000000..b215725594
--- /dev/null
+++ b/target/arm/sme_helper.c
@@ -0,0 +1,61 @@
+/*
+ * ARM SME Operations
+ *
+ * Copyright (c) 2022 Linaro, Ltd.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "cpu.h"
+#include "internals.h"
+#include "exec/helper-proto.h"
+
+/* ResetSVEState */
+void arm_reset_sve_state(CPUARMState *env)
+{
+ memset(env->vfp.zregs, 0, sizeof(env->vfp.zregs));
+ /* Recall that FFR is stored as pregs[16]. */
+ memset(env->vfp.pregs, 0, sizeof(env->vfp.pregs));
+ vfp_set_fpcr(env, 0x0800009f);
+}
+
+void helper_set_pstate_sm(CPUARMState *env, uint32_t i)
+{
+ if (i == FIELD_EX64(env->svcr, SVCR, SM)) {
+ return;
+ }
+ env->svcr ^= R_SVCR_SM_MASK;
+ arm_reset_sve_state(env);
+}
+
+void helper_set_pstate_za(CPUARMState *env, uint32_t i)
+{
+ if (i == FIELD_EX64(env->svcr, SVCR, ZA)) {
+ return;
+ }
+ env->svcr ^= R_SVCR_ZA_MASK;
+
+ /*
+ * ResetSMEState.
+ *
+ * SetPSTATE_ZA zeros on enable and disable. We can zero this only
+ * on enable: while disabled, the storage is inaccessible and the
+ * value does not matter. We're not saving the storage in vmstate
+ * when disabled either.
+ */
+ if (i) {
+ memset(env->zarray, 0, sizeof(env->zarray));
+ }
+}
diff --git a/target/arm/syndrome.h b/target/arm/syndrome.h
index c105f9e6ba..73df5e3793 100644
--- a/target/arm/syndrome.h
+++ b/target/arm/syndrome.h
@@ -48,6 +48,7 @@ enum arm_exception_class {
EC_AA64_SMC = 0x17,
EC_SYSTEMREGISTERTRAP = 0x18,
EC_SVEACCESSTRAP = 0x19,
+ EC_SMETRAP = 0x1d,
EC_INSNABORT = 0x20,
EC_INSNABORT_SAME_EL = 0x21,
EC_PCALIGNMENT = 0x22,
@@ -68,6 +69,13 @@ enum arm_exception_class {
EC_AA64_BKPT = 0x3c,
};
+typedef enum {
+ SME_ET_AccessTrap,
+ SME_ET_Streaming,
+ SME_ET_NotStreaming,
+ SME_ET_InactiveZA,
+} SMEExceptionType;
+
#define ARM_EL_EC_SHIFT 26
#define ARM_EL_IL_SHIFT 25
#define ARM_EL_ISV_SHIFT 24
@@ -207,6 +215,12 @@ static inline uint32_t syn_sve_access_trap(void)
return EC_SVEACCESSTRAP << ARM_EL_EC_SHIFT;
}
+static inline uint32_t syn_smetrap(SMEExceptionType etype, bool is_16bit)
+{
+ return (EC_SMETRAP << ARM_EL_EC_SHIFT)
+ | (is_16bit ? 0 : ARM_EL_IL) | etype;
+}
+
static inline uint32_t syn_pactrap(void)
{
return EC_PACTRAP << ARM_EL_EC_SHIFT;
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index 4c64546090..c86b97b1d4 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -1188,6 +1188,22 @@ bool sve_access_check(DisasContext *s)
}
/*
+ * Check that SME access is enabled, raise an exception if not.
+ * Note that this function corresponds to CheckSMEAccess and is
+ * only used directly for cpregs.
+ */
+static bool sme_access_check(DisasContext *s)
+{
+ if (s->sme_excp_el) {
+ gen_exception_insn_el(s, s->pc_curr, EXCP_UDEF,
+ syn_smetrap(SME_ET_AccessTrap, false),
+ s->sme_excp_el);
+ return false;
+ }
+ return true;
+}
+
+/*
* This utility function is for doing register extension with an
* optional shift. You will likely want to pass a temporary for the
* destination register. See DecodeRegExtend() in the ARM ARM.
@@ -1746,6 +1762,30 @@ static void handle_msr_i(DisasContext *s, uint32_t insn,
}
break;
+ case 0x1b: /* SVCR* */
+ if (!dc_isar_feature(aa64_sme, s) || crm < 2 || crm > 7) {
+ goto do_unallocated;
+ }
+ if (sme_access_check(s)) {
+ bool i = crm & 1;
+ bool changed = false;
+
+ if ((crm & 2) && i != s->pstate_sm) {
+ gen_helper_set_pstate_sm(cpu_env, tcg_constant_i32(i));
+ changed = true;
+ }
+ if ((crm & 4) && i != s->pstate_za) {
+ gen_helper_set_pstate_za(cpu_env, tcg_constant_i32(i));
+ changed = true;
+ }
+ if (changed) {
+ gen_rebuild_hflags(s);
+ } else {
+ s->base.is_jmp = DISAS_NEXT;
+ }
+ }
+ break;
+
default:
do_unallocated:
unallocated_encoding(s);
@@ -1958,6 +1998,8 @@ static void handle_sys(DisasContext *s, uint32_t insn, bool isread,
return;
} else if ((ri->type & ARM_CP_SVE) && !sve_access_check(s)) {
return;
+ } else if ((ri->type & ARM_CP_SME) && !sme_access_check(s)) {
+ return;
}
if ((tb_cflags(s->base.tb) & CF_USE_ICOUNT) && (ri->type & ARM_CP_IO)) {
@@ -14603,7 +14645,9 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM);
dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL);
dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL);
+ dc->sme_excp_el = EX_TBFLAG_A64(tb_flags, SMEEXC_EL);
dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16;
+ dc->svl = (EX_TBFLAG_A64(tb_flags, SVL) + 1) * 16;
dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE);
dc->bt = EX_TBFLAG_A64(tb_flags, BT);
dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE);
@@ -14611,6 +14655,8 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase,
dc->ata = EX_TBFLAG_A64(tb_flags, ATA);
dc->mte_active[0] = EX_TBFLAG_A64(tb_flags, MTE_ACTIVE);
dc->mte_active[1] = EX_TBFLAG_A64(tb_flags, MTE0_ACTIVE);
+ dc->pstate_sm = EX_TBFLAG_A64(tb_flags, PSTATE_SM);
+ dc->pstate_za = EX_TBFLAG_A64(tb_flags, PSTATE_ZA);
dc->vec_len = 0;
dc->vec_stride = 0;
dc->cp_regs = arm_cpu->cp_regs;
diff --git a/target/arm/translate-a64.h b/target/arm/translate-a64.h
index dbc917ee65..f0970c6b8c 100644
--- a/target/arm/translate-a64.h
+++ b/target/arm/translate-a64.h
@@ -107,6 +107,44 @@ static inline int vec_full_reg_size(DisasContext *s)
return s->vl;
}
+/*
+ * Return the offset info CPUARMState of the predicate vector register Pn.
+ * Note for this purpose, FFR is P16.
+ */
+static inline int pred_full_reg_offset(DisasContext *s, int regno)
+{
+ return offsetof(CPUARMState, vfp.pregs[regno]);
+}
+
+/* Return the byte size of the whole predicate register, VL / 64. */
+static inline int pred_full_reg_size(DisasContext *s)
+{
+ return s->vl >> 3;
+}
+
+/*
+ * Round up the size of a register to a size allowed by
+ * the tcg vector infrastructure. Any operation which uses this
+ * size may assume that the bits above pred_full_reg_size are zero,
+ * and must leave them the same way.
+ *
+ * Note that this is not needed for the vector registers as they
+ * are always properly sized for tcg vectors.
+ */
+static inline int size_for_gvec(int size)
+{
+ if (size <= 8) {
+ return 8;
+ } else {
+ return QEMU_ALIGN_UP(size, 16);
+ }
+}
+
+static inline int pred_gvec_reg_size(DisasContext *s)
+{
+ return size_for_gvec(pred_full_reg_size(s));
+}
+
bool disas_sve(DisasContext *, uint32_t);
void gen_gvec_rax1(unsigned vece, uint32_t rd_ofs, uint32_t rn_ofs,
diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c
index 67761bf2cc..62b5f3040c 100644
--- a/target/arm/translate-sve.c
+++ b/target/arm/translate-sve.c
@@ -100,42 +100,6 @@ static inline int msz_dtype(DisasContext *s, int msz)
* Implement all of the translator functions referenced by the decoder.
*/
-/* Return the offset info CPUARMState of the predicate vector register Pn.
- * Note for this purpose, FFR is P16.
- */
-static inline int pred_full_reg_offset(DisasContext *s, int regno)
-{
- return offsetof(CPUARMState, vfp.pregs[regno]);
-}
-
-/* Return the byte size of the whole predicate register, VL / 64. */
-static inline int pred_full_reg_size(DisasContext *s)
-{
- return s->vl >> 3;
-}
-
-/* Round up the size of a register to a size allowed by
- * the tcg vector infrastructure. Any operation which uses this
- * size may assume that the bits above pred_full_reg_size are zero,
- * and must leave them the same way.
- *
- * Note that this is not needed for the vector registers as they
- * are always properly sized for tcg vectors.
- */
-static int size_for_gvec(int size)
-{
- if (size <= 8) {
- return 8;
- } else {
- return QEMU_ALIGN_UP(size, 16);
- }
-}
-
-static int pred_gvec_reg_size(DisasContext *s)
-{
- return size_for_gvec(pred_full_reg_size(s));
-}
-
/* Invoke an out-of-line helper on 2 Zregs. */
static bool gen_gvec_ool_zz(DisasContext *s, gen_helper_gvec_2 *fn,
int rd, int rn, int data)
diff --git a/target/arm/translate.h b/target/arm/translate.h
index 88dc18a034..22fd882368 100644
--- a/target/arm/translate.h
+++ b/target/arm/translate.h
@@ -42,7 +42,9 @@ typedef struct DisasContext {
bool ns; /* Use non-secure CPREG bank on access */
int fp_excp_el; /* FP exception EL or 0 if enabled */
int sve_excp_el; /* SVE exception EL or 0 if enabled */
+ int sme_excp_el; /* SME exception EL or 0 if enabled */
int vl; /* current vector length in bytes */
+ int svl; /* current streaming vector length in bytes */
bool vfp_enabled; /* FP enabled via FPSCR.EN */
int vec_len;
int vec_stride;
@@ -96,6 +98,10 @@ typedef struct DisasContext {
bool align_mem;
/* True if PSTATE.IL is set */
bool pstate_il;
+ /* True if PSTATE.SM is set. */
+ bool pstate_sm;
+ /* True if PSTATE.ZA is set. */
+ bool pstate_za;
/* True if MVE insns are definitely not predicated by VPR or LTPSIZE */
bool mve_no_pred;
/*
generated by cgit v1.2.3-55-g7522 (git 2.39.0) at 2025-04-27 04:56:08 +0200