aboutsummaryrefslogtreecommitdiff
path: root/target-arm/helper.c
diff options
context:
space:
mode:
Diffstat (limited to 'target-arm/helper.c')
-rw-r--r--target-arm/helper.c810
1 files changed, 617 insertions, 193 deletions
diff --git a/target-arm/helper.c b/target-arm/helper.c
index 55077ed1b6..43c1b4f01d 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -1,4 +1,5 @@
#include "cpu.h"
+#include "internals.h"
#include "exec/gdbstub.h"
#include "helper.h"
#include "qemu/host-utils.h"
@@ -9,7 +10,9 @@
#include <zlib.h> /* For crc32 */
#ifndef CONFIG_USER_ONLY
-static inline int get_phys_addr(CPUARMState *env, uint32_t address,
+#include "exec/softmmu_exec.h"
+
+static inline int get_phys_addr(CPUARMState *env, target_ulong address,
int access_type, int is_user,
hwaddr *phys_ptr, int *prot,
target_ulong *page_size);
@@ -301,6 +304,17 @@ void init_cpreg_list(ARMCPU *cpu)
g_list_free(keys);
}
+/* Return true if extended addresses are enabled.
+ * This is always the case if our translation regime is 64 bit,
+ * but depends on TTBCR.EAE for 32 bit.
+ */
+static inline bool extended_addresses_enabled(CPUARMState *env)
+{
+ return arm_el_is_aa64(env, 1)
+ || ((arm_feature(env, ARM_FEATURE_LPAE)
+ && (env->cp15.c2_control & (1U << 31))));
+}
+
static void dacr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
{
ARMCPU *cpu = arm_env_get_cpu(env);
@@ -327,14 +341,15 @@ static void contextidr_write(CPUARMState *env, const ARMCPRegInfo *ri,
{
ARMCPU *cpu = arm_env_get_cpu(env);
- if (env->cp15.c13_context != value && !arm_feature(env, ARM_FEATURE_MPU)) {
+ if (env->cp15.contextidr_el1 != value && !arm_feature(env, ARM_FEATURE_MPU)
+ && !extended_addresses_enabled(env)) {
/* For VMSA (when not using the LPAE long descriptor page table
* format) this register includes the ASID, so do a TLB flush.
* For PMSA it is purely a process ID and no action is needed.
*/
tlb_flush(CPU(cpu), 1);
}
- env->cp15.c13_context = value;
+ env->cp15.contextidr_el1 = value;
}
static void tlbiall_write(CPUARMState *env, const ARMCPRegInfo *ri,
@@ -380,17 +395,26 @@ static const ARMCPRegInfo cp_reginfo[] = {
*/
{ .name = "DBGDIDR", .cp = 14, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 0,
.access = PL0_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+ { .name = "FCSEIDR", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 0,
+ .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c13_fcse),
+ .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
+ { .name = "CONTEXTIDR", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 13, .crm = 0, .opc2 = 1,
+ .access = PL1_RW,
+ .fieldoffset = offsetof(CPUARMState, cp15.contextidr_el1),
+ .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
+ REGINFO_SENTINEL
+};
+
+static const ARMCPRegInfo not_v8_cp_reginfo[] = {
+ /* NB: Some of these registers exist in v8 but with more precise
+ * definitions that don't use CP_ANY wildcards (mostly in v8_cp_reginfo[]).
+ */
/* MMU Domain access control / MPU write buffer control */
{ .name = "DACR", .cp = 15,
.crn = 3, .crm = CP_ANY, .opc1 = CP_ANY, .opc2 = CP_ANY,
.access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c3),
.resetvalue = 0, .writefn = dacr_write, .raw_writefn = raw_write, },
- { .name = "FCSEIDR", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 0,
- .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c13_fcse),
- .resetvalue = 0, .writefn = fcse_write, .raw_writefn = raw_write, },
- { .name = "CONTEXTIDR", .cp = 15, .crn = 13, .crm = 0, .opc1 = 0, .opc2 = 1,
- .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c13_context),
- .resetvalue = 0, .writefn = contextidr_write, .raw_writefn = raw_write, },
/* ??? This covers not just the impdef TLB lockdown registers but also
* some v7VMSA registers relating to TEX remap, so it is overly broad.
*/
@@ -472,7 +496,8 @@ static const ARMCPRegInfo v6_cp_reginfo[] = {
{ .name = "DMB", .cp = 15, .crn = 7, .crm = 10, .opc1 = 0, .opc2 = 5,
.access = PL0_W, .type = ARM_CP_NOP },
{ .name = "IFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 2,
- .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c6_insn),
+ .access = PL1_RW,
+ .fieldoffset = offsetofhigh32(CPUARMState, cp15.far_el1),
.resetvalue = 0, },
/* Watchpoint Fault Address Register : should actually only be present
* for 1136, 1176, 11MPCore.
@@ -647,6 +672,21 @@ static void csselr_write(CPUARMState *env, const ARMCPRegInfo *ri,
env->cp15.c0_cssel = value & 0xf;
}
+static uint64_t isr_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+ CPUState *cs = ENV_GET_CPU(env);
+ uint64_t ret = 0;
+
+ if (cs->interrupt_request & CPU_INTERRUPT_HARD) {
+ ret |= CPSR_I;
+ }
+ if (cs->interrupt_request & CPU_INTERRUPT_FIQ) {
+ ret |= CPSR_F;
+ }
+ /* External aborts are not possible in QEMU so A bit is always clear */
+ return ret;
+}
+
static const ARMCPRegInfo v7_cp_reginfo[] = {
/* DBGDRAR, DBGDSAR: always RAZ since we don't implement memory mapped
* debug components
@@ -741,8 +781,18 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
/* Auxiliary ID register: this actually has an IMPDEF value but for now
* just RAZ for all cores:
*/
- { .name = "AIDR", .cp = 15, .crn = 0, .crm = 0, .opc1 = 1, .opc2 = 7,
+ { .name = "AIDR", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 1, .crn = 0, .crm = 0, .opc2 = 7,
.access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+ /* Auxiliary fault status registers: these also are IMPDEF, and we
+ * choose to RAZ/WI for all cores.
+ */
+ { .name = "AFSR0_EL1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 0,
+ .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
+ { .name = "AFSR1_EL1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 5, .crm = 1, .opc2 = 1,
+ .access = PL1_RW, .type = ARM_CP_CONST, .resetvalue = 0 },
/* MAIR can just read-as-written because we don't implement caches
* and so don't need to care about memory attributes.
*/
@@ -763,6 +813,9 @@ static const ARMCPRegInfo v7_cp_reginfo[] = {
.cp = 15, .opc1 = 0, .crn = 10, .crm = 2, .opc2 = 1, .access = PL1_RW,
.fieldoffset = offsetofhigh32(CPUARMState, cp15.mair_el1),
.resetfn = arm_cp_reset_ignore },
+ { .name = "ISR_EL1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 1, .opc2 = 0,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_R, .readfn = isr_read },
REGINFO_SENTINEL
};
@@ -1139,27 +1192,17 @@ static const ARMCPRegInfo generic_timer_cp_reginfo[] = {
static void par_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
{
if (arm_feature(env, ARM_FEATURE_LPAE)) {
- env->cp15.c7_par = value;
+ env->cp15.par_el1 = value;
} else if (arm_feature(env, ARM_FEATURE_V7)) {
- env->cp15.c7_par = value & 0xfffff6ff;
+ env->cp15.par_el1 = value & 0xfffff6ff;
} else {
- env->cp15.c7_par = value & 0xfffff1ff;
+ env->cp15.par_el1 = value & 0xfffff1ff;
}
}
#ifndef CONFIG_USER_ONLY
/* get_phys_addr() isn't present for user-mode-only targets */
-/* Return true if extended addresses are enabled, ie this is an
- * LPAE implementation and we are using the long-descriptor translation
- * table format because the TTBCR EAE bit is set.
- */
-static inline bool extended_addresses_enabled(CPUARMState *env)
-{
- return arm_feature(env, ARM_FEATURE_LPAE)
- && (env->cp15.c2_control & (1U << 31));
-}
-
static CPAccessResult ats_access(CPUARMState *env, const ARMCPRegInfo *ri)
{
if (ri->opc2 & 4) {
@@ -1200,8 +1243,7 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
* fault.
*/
}
- env->cp15.c7_par = par64;
- env->cp15.c7_par_hi = par64 >> 32;
+ env->cp15.par_el1 = par64;
} else {
/* ret is a DFSR/IFSR value for the short descriptor
* translation table format (with WnR always clear).
@@ -1211,16 +1253,15 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
/* We do not set any attribute bits in the PAR */
if (page_size == (1 << 24)
&& arm_feature(env, ARM_FEATURE_V7)) {
- env->cp15.c7_par = (phys_addr & 0xff000000) | 1 << 1;
+ env->cp15.par_el1 = (phys_addr & 0xff000000) | 1 << 1;
} else {
- env->cp15.c7_par = phys_addr & 0xfffff000;
+ env->cp15.par_el1 = phys_addr & 0xfffff000;
}
} else {
- env->cp15.c7_par = ((ret & (1 << 10)) >> 5) |
+ env->cp15.par_el1 = ((ret & (1 << 10)) >> 5) |
((ret & (1 << 12)) >> 6) |
((ret & 0xf) << 1) | 1;
}
- env->cp15.c7_par_hi = 0;
}
}
#endif
@@ -1228,7 +1269,7 @@ static void ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
static const ARMCPRegInfo vapa_cp_reginfo[] = {
{ .name = "PAR", .cp = 15, .crn = 7, .crm = 4, .opc1 = 0, .opc2 = 0,
.access = PL1_RW, .resetvalue = 0,
- .fieldoffset = offsetof(CPUARMState, cp15.c7_par),
+ .fieldoffset = offsetoflow32(CPUARMState, cp15.par_el1),
.writefn = par_write },
#ifndef CONFIG_USER_ONLY
{ .name = "ATS", .cp = 15, .crn = 7, .crm = 8, .opc1 = 0, .opc2 = CP_ANY,
@@ -1271,40 +1312,44 @@ static uint32_t extended_mpu_ap_bits(uint32_t val)
static void pmsav5_data_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
uint64_t value)
{
- env->cp15.c5_data = extended_mpu_ap_bits(value);
+ env->cp15.pmsav5_data_ap = extended_mpu_ap_bits(value);
}
static uint64_t pmsav5_data_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
{
- return simple_mpu_ap_bits(env->cp15.c5_data);
+ return simple_mpu_ap_bits(env->cp15.pmsav5_data_ap);
}
static void pmsav5_insn_ap_write(CPUARMState *env, const ARMCPRegInfo *ri,
uint64_t value)
{
- env->cp15.c5_insn = extended_mpu_ap_bits(value);
+ env->cp15.pmsav5_insn_ap = extended_mpu_ap_bits(value);
}
static uint64_t pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri)
{
- return simple_mpu_ap_bits(env->cp15.c5_insn);
+ return simple_mpu_ap_bits(env->cp15.pmsav5_insn_ap);
}
static const ARMCPRegInfo pmsav5_cp_reginfo[] = {
{ .name = "DATA_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
.access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
- .fieldoffset = offsetof(CPUARMState, cp15.c5_data), .resetvalue = 0,
+ .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
+ .resetvalue = 0,
.readfn = pmsav5_data_ap_read, .writefn = pmsav5_data_ap_write, },
{ .name = "INSN_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
.access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
- .fieldoffset = offsetof(CPUARMState, cp15.c5_insn), .resetvalue = 0,
+ .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
+ .resetvalue = 0,
.readfn = pmsav5_insn_ap_read, .writefn = pmsav5_insn_ap_write, },
{ .name = "DATA_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 2,
.access = PL1_RW,
- .fieldoffset = offsetof(CPUARMState, cp15.c5_data), .resetvalue = 0, },
+ .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_data_ap),
+ .resetvalue = 0, },
{ .name = "INSN_EXT_AP", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 3,
.access = PL1_RW,
- .fieldoffset = offsetof(CPUARMState, cp15.c5_insn), .resetvalue = 0, },
+ .fieldoffset = offsetof(CPUARMState, cp15.pmsav5_insn_ap),
+ .resetvalue = 0, },
{ .name = "DCACHE_CFG", .cp = 15, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
.access = PL1_RW,
.fieldoffset = offsetof(CPUARMState, cp15.c2_data), .resetvalue = 0, },
@@ -1406,11 +1451,16 @@ static void vmsa_ttbr_write(CPUARMState *env, const ARMCPRegInfo *ri,
static const ARMCPRegInfo vmsa_cp_reginfo[] = {
{ .name = "DFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 0,
- .access = PL1_RW,
- .fieldoffset = offsetof(CPUARMState, cp15.c5_data), .resetvalue = 0, },
+ .access = PL1_RW, .type = ARM_CP_NO_MIGRATE,
+ .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el1),
+ .resetfn = arm_cp_reset_ignore, },
{ .name = "IFSR", .cp = 15, .crn = 5, .crm = 0, .opc1 = 0, .opc2 = 1,
.access = PL1_RW,
- .fieldoffset = offsetof(CPUARMState, cp15.c5_insn), .resetvalue = 0, },
+ .fieldoffset = offsetof(CPUARMState, cp15.ifsr_el2), .resetvalue = 0, },
+ { .name = "ESR_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .crn = 5, .crm = 2, .opc1 = 0, .opc2 = 0,
+ .access = PL1_RW,
+ .fieldoffset = offsetof(CPUARMState, cp15.esr_el1), .resetvalue = 0, },
{ .name = "TTBR0_EL1", .state = ARM_CP_STATE_BOTH,
.opc0 = 3, .crn = 2, .crm = 0, .opc1 = 0, .opc2 = 0,
.access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el1),
@@ -1428,8 +1478,10 @@ static const ARMCPRegInfo vmsa_cp_reginfo[] = {
.access = PL1_RW, .type = ARM_CP_NO_MIGRATE, .writefn = vmsa_ttbcr_write,
.resetfn = arm_cp_reset_ignore, .raw_writefn = vmsa_ttbcr_raw_write,
.fieldoffset = offsetoflow32(CPUARMState, cp15.c2_control) },
- { .name = "DFAR", .cp = 15, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
- .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c6_data),
+ /* 64-bit FAR; this entry also gives us the AArch32 DFAR */
+ { .name = "FAR_EL1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .crn = 6, .crm = 0, .opc1 = 0, .opc2 = 0,
+ .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.far_el1),
.resetvalue = 0, },
REGINFO_SENTINEL
};
@@ -1469,7 +1521,8 @@ static void omap_cachemaint_write(CPUARMState *env, const ARMCPRegInfo *ri,
static const ARMCPRegInfo omap_cp_reginfo[] = {
{ .name = "DFSR", .cp = 15, .crn = 5, .crm = CP_ANY,
.opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_RW, .type = ARM_CP_OVERRIDE,
- .fieldoffset = offsetof(CPUARMState, cp15.c5_data), .resetvalue = 0, },
+ .fieldoffset = offsetoflow32(CPUARMState, cp15.esr_el1),
+ .resetvalue = 0, },
{ .name = "", .cp = 15, .crn = 15, .crm = 0, .opc1 = 0, .opc2 = 0,
.access = PL1_RW, .type = ARM_CP_NOP },
{ .name = "TICONFIG", .cp = 15, .crn = 15, .crm = 1, .opc1 = 0, .opc2 = 0,
@@ -1619,24 +1672,6 @@ static const ARMCPRegInfo mpidr_cp_reginfo[] = {
REGINFO_SENTINEL
};
-static uint64_t par64_read(CPUARMState *env, const ARMCPRegInfo *ri)
-{
- return ((uint64_t)env->cp15.c7_par_hi << 32) | env->cp15.c7_par;
-}
-
-static void par64_write(CPUARMState *env, const ARMCPRegInfo *ri,
- uint64_t value)
-{
- env->cp15.c7_par_hi = value >> 32;
- env->cp15.c7_par = value;
-}
-
-static void par64_reset(CPUARMState *env, const ARMCPRegInfo *ri)
-{
- env->cp15.c7_par_hi = 0;
- env->cp15.c7_par = 0;
-}
-
static const ARMCPRegInfo lpae_cp_reginfo[] = {
/* NOP AMAIR0/1: the override is because these clash with the rather
* broadly specified TLB_LOCKDOWN entry in the generic cp_reginfo.
@@ -1656,7 +1691,7 @@ static const ARMCPRegInfo lpae_cp_reginfo[] = {
.access = PL0_R, .type = ARM_CP_CONST|ARM_CP_64BIT, .resetvalue = 0 },
{ .name = "PAR", .cp = 15, .crm = 7, .opc1 = 0,
.access = PL1_RW, .type = ARM_CP_64BIT,
- .readfn = par64_read, .writefn = par64_write, .resetfn = par64_reset },
+ .fieldoffset = offsetof(CPUARMState, cp15.par_el1), .resetvalue = 0 },
{ .name = "TTBR0", .cp = 15, .crm = 2, .opc1 = 0,
.access = PL1_RW, .type = ARM_CP_64BIT | ARM_CP_NO_MIGRATE,
.fieldoffset = offsetof(CPUARMState, cp15.ttbr0_el1),
@@ -1690,6 +1725,20 @@ static void aa64_fpsr_write(CPUARMState *env, const ARMCPRegInfo *ri,
vfp_set_fpsr(env, value);
}
+static CPAccessResult aa64_daif_access(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+ if (arm_current_pl(env) == 0 && !(env->cp15.c1_sys & SCTLR_UMA)) {
+ return CP_ACCESS_TRAP;
+ }
+ return CP_ACCESS_OK;
+}
+
+static void aa64_daif_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ env->daif = value & PSTATE_DAIF;
+}
+
static CPAccessResult aa64_cacheop_access(CPUARMState *env,
const ARMCPRegInfo *ri)
{
@@ -1729,6 +1778,50 @@ static void tlbi_aa64_asid_write(CPUARMState *env, const ARMCPRegInfo *ri,
tlb_flush(CPU(cpu), asid == 0);
}
+static CPAccessResult aa64_zva_access(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+ /* We don't implement EL2, so the only control on DC ZVA is the
+ * bit in the SCTLR which can prohibit access for EL0.
+ */
+ if (arm_current_pl(env) == 0 && !(env->cp15.c1_sys & SCTLR_DZE)) {
+ return CP_ACCESS_TRAP;
+ }
+ return CP_ACCESS_OK;
+}
+
+static uint64_t aa64_dczid_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+ ARMCPU *cpu = arm_env_get_cpu(env);
+ int dzp_bit = 1 << 4;
+
+ /* DZP indicates whether DC ZVA access is allowed */
+ if (aa64_zva_access(env, NULL) != CP_ACCESS_OK) {
+ dzp_bit = 0;
+ }
+ return cpu->dcz_blocksize | dzp_bit;
+}
+
+static CPAccessResult sp_el0_access(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+ if (!env->pstate & PSTATE_SP) {
+ /* Access to SP_EL0 is undefined if it's being used as
+ * the stack pointer.
+ */
+ return CP_ACCESS_TRAP_UNCATEGORIZED;
+ }
+ return CP_ACCESS_OK;
+}
+
+static uint64_t spsel_read(CPUARMState *env, const ARMCPRegInfo *ri)
+{
+ return env->pstate & PSTATE_SP;
+}
+
+static void spsel_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t val)
+{
+ update_spsel(env, val);
+}
+
static const ARMCPRegInfo v8_cp_reginfo[] = {
/* Minimal set of EL0-visible registers. This will need to be expanded
* significantly for system emulation of AArch64 CPUs.
@@ -1736,19 +1829,30 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
{ .name = "NZCV", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 2,
.access = PL0_RW, .type = ARM_CP_NZCV },
+ { .name = "DAIF", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 2,
+ .type = ARM_CP_NO_MIGRATE,
+ .access = PL0_RW, .accessfn = aa64_daif_access,
+ .fieldoffset = offsetof(CPUARMState, daif),
+ .writefn = aa64_daif_write, .resetfn = arm_cp_reset_ignore },
{ .name = "FPCR", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 3, .opc2 = 0, .crn = 4, .crm = 4,
.access = PL0_RW, .readfn = aa64_fpcr_read, .writefn = aa64_fpcr_write },
{ .name = "FPSR", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 4, .crm = 4,
.access = PL0_RW, .readfn = aa64_fpsr_read, .writefn = aa64_fpsr_write },
- /* Prohibit use of DC ZVA. OPTME: implement DC ZVA and allow its use.
- * For system mode the DZP bit here will need to be computed, not constant.
- */
{ .name = "DCZID_EL0", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 3, .opc2 = 7, .crn = 0, .crm = 0,
- .access = PL0_R, .type = ARM_CP_CONST,
- .resetvalue = 0x10 },
+ .access = PL0_R, .type = ARM_CP_NO_MIGRATE,
+ .readfn = aa64_dczid_read },
+ { .name = "DC_ZVA", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 3, .crn = 7, .crm = 4, .opc2 = 1,
+ .access = PL0_W, .type = ARM_CP_DC_ZVA,
+#ifndef CONFIG_USER_ONLY
+ /* Avoid overhead of an access check that always passes in user-mode */
+ .accessfn = aa64_zva_access,
+#endif
+ },
{ .name = "CURRENTEL", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .opc2 = 2, .crn = 4, .crm = 2,
.access = PL1_R, .type = ARM_CP_CURRENTEL },
@@ -1836,6 +1940,93 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
.opc0 = 1, .opc2 = 0, .crn = 8, .crm = 7, .opc2 = 7,
.access = PL1_W, .type = ARM_CP_NO_MIGRATE,
.writefn = tlbi_aa64_vaa_write },
+#ifndef CONFIG_USER_ONLY
+ /* 64 bit address translation operations */
+ { .name = "AT_S1E1R", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 0,
+ .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
+ { .name = "AT_S1E1W", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 1,
+ .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
+ { .name = "AT_S1E0R", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 2,
+ .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
+ { .name = "AT_S1E0W", .state = ARM_CP_STATE_AA64,
+ .opc0 = 1, .opc1 = 0, .crn = 7, .crm = 8, .opc2 = 3,
+ .access = PL1_W, .type = ARM_CP_NO_MIGRATE, .writefn = ats_write },
+#endif
+ /* 32 bit TLB invalidates, Inner Shareable */
+ { .name = "TLBIALLIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 0,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+ { .name = "TLBIMVAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 1,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+ { .name = "TLBIASIDIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 2,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+ { .name = "TLBIMVAAIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 3,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+ { .name = "TLBIMVALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 5,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+ { .name = "TLBIMVAALIS", .cp = 15, .opc1 = 0, .crn = 8, .crm = 3, .opc2 = 7,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+ /* 32 bit ITLB invalidates */
+ { .name = "ITLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 0,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+ { .name = "ITLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 1,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+ { .name = "ITLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 5, .opc2 = 2,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+ /* 32 bit DTLB invalidates */
+ { .name = "DTLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 0,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+ { .name = "DTLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 1,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+ { .name = "DTLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 6, .opc2 = 2,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+ /* 32 bit TLB invalidates */
+ { .name = "TLBIALL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 0,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiall_write },
+ { .name = "TLBIMVA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 1,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+ { .name = "TLBIASID", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 2,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbiasid_write },
+ { .name = "TLBIMVAA", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 3,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+ { .name = "TLBIMVAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 5,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimva_write },
+ { .name = "TLBIMVAAL", .cp = 15, .opc1 = 0, .crn = 8, .crm = 7, .opc2 = 7,
+ .type = ARM_CP_NO_MIGRATE, .access = PL1_W, .writefn = tlbimvaa_write },
+ /* 32 bit cache operations */
+ { .name = "ICIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 0,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "BPIALLUIS", .cp = 15, .opc1 = 0, .crn = 7, .crm = 1, .opc2 = 6,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "ICIALLU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 0,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "ICIMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 1,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "BPIALL", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 6,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "BPIMVA", .cp = 15, .opc1 = 0, .crn = 7, .crm = 5, .opc2 = 7,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "DCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 1,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "DCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 6, .opc2 = 2,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "DCCMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 1,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "DCCSW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 10, .opc2 = 2,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "DCCMVAU", .cp = 15, .opc1 = 0, .crn = 7, .crm = 11, .opc2 = 1,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "DCCIMVAC", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 1,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ { .name = "DCCISW", .cp = 15, .opc1 = 0, .crn = 7, .crm = 14, .opc2 = 2,
+ .type = ARM_CP_NOP, .access = PL1_W },
+ /* MMU Domain access control / MPU write buffer control */
+ { .name = "DACR", .cp = 15,
+ .opc1 = 0, .crn = 3, .crm = 0, .opc2 = 0,
+ .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, cp15.c3),
+ .resetvalue = 0, .writefn = dacr_write, .raw_writefn = raw_write, },
/* Dummy implementation of monitor debug system control register:
* we don't support debug.
*/
@@ -1846,6 +2037,27 @@ static const ARMCPRegInfo v8_cp_reginfo[] = {
{ .name = "OSLAR_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 2, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 4,
.access = PL1_W, .type = ARM_CP_NOP },
+ { .name = "ELR_EL1", .state = ARM_CP_STATE_AA64,
+ .type = ARM_CP_NO_MIGRATE,
+ .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 1,
+ .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, elr_el1) },
+ { .name = "SPSR_EL1", .state = ARM_CP_STATE_AA64,
+ .type = ARM_CP_NO_MIGRATE,
+ .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 0, .opc2 = 0,
+ .access = PL1_RW, .fieldoffset = offsetof(CPUARMState, banked_spsr[0]) },
+ /* We rely on the access checks not allowing the guest to write to the
+ * state field when SPSel indicates that it's being used as the stack
+ * pointer.
+ */
+ { .name = "SP_EL0", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 1, .opc2 = 0,
+ .access = PL1_RW, .accessfn = sp_el0_access,
+ .type = ARM_CP_NO_MIGRATE,
+ .fieldoffset = offsetof(CPUARMState, sp_el[0]) },
+ { .name = "SPSel", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 4, .crm = 2, .opc2 = 0,
+ .type = ARM_CP_NO_MIGRATE,
+ .access = PL1_RW, .readfn = spsel_read, .writefn = spsel_write },
REGINFO_SENTINEL
};
@@ -1912,50 +2124,71 @@ void register_cp_regs_for_features(ARMCPU *cpu)
}
define_arm_cp_regs(cpu, cp_reginfo);
+ if (!arm_feature(env, ARM_FEATURE_V8)) {
+ /* Must go early as it is full of wildcards that may be
+ * overridden by later definitions.
+ */
+ define_arm_cp_regs(cpu, not_v8_cp_reginfo);
+ }
+
if (arm_feature(env, ARM_FEATURE_V6)) {
/* The ID registers all have impdef reset values */
ARMCPRegInfo v6_idregs[] = {
- { .name = "ID_PFR0", .cp = 15, .crn = 0, .crm = 1,
- .opc1 = 0, .opc2 = 0, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_PFR0", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 0,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_pfr0 },
- { .name = "ID_PFR1", .cp = 15, .crn = 0, .crm = 1,
- .opc1 = 0, .opc2 = 1, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_PFR1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 1,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_pfr1 },
- { .name = "ID_DFR0", .cp = 15, .crn = 0, .crm = 1,
- .opc1 = 0, .opc2 = 2, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_DFR0", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 2,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_dfr0 },
- { .name = "ID_AFR0", .cp = 15, .crn = 0, .crm = 1,
- .opc1 = 0, .opc2 = 3, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_AFR0", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 3,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_afr0 },
- { .name = "ID_MMFR0", .cp = 15, .crn = 0, .crm = 1,
- .opc1 = 0, .opc2 = 4, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_MMFR0", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 4,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_mmfr0 },
- { .name = "ID_MMFR1", .cp = 15, .crn = 0, .crm = 1,
- .opc1 = 0, .opc2 = 5, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_MMFR1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 5,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_mmfr1 },
- { .name = "ID_MMFR2", .cp = 15, .crn = 0, .crm = 1,
- .opc1 = 0, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_MMFR2", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 6,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_mmfr2 },
- { .name = "ID_MMFR3", .cp = 15, .crn = 0, .crm = 1,
- .opc1 = 0, .opc2 = 7, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_MMFR3", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 1, .opc2 = 7,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_mmfr3 },
- { .name = "ID_ISAR0", .cp = 15, .crn = 0, .crm = 2,
- .opc1 = 0, .opc2 = 0, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_ISAR0", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 0,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_isar0 },
- { .name = "ID_ISAR1", .cp = 15, .crn = 0, .crm = 2,
- .opc1 = 0, .opc2 = 1, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_ISAR1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 1,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_isar1 },
- { .name = "ID_ISAR2", .cp = 15, .crn = 0, .crm = 2,
- .opc1 = 0, .opc2 = 2, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_ISAR2", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 2,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_isar2 },
- { .name = "ID_ISAR3", .cp = 15, .crn = 0, .crm = 2,
- .opc1 = 0, .opc2 = 3, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_ISAR3", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 3,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_isar3 },
- { .name = "ID_ISAR4", .cp = 15, .crn = 0, .crm = 2,
- .opc1 = 0, .opc2 = 4, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_ISAR4", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 4,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_isar4 },
- { .name = "ID_ISAR5", .cp = 15, .crn = 0, .crm = 2,
- .opc1 = 0, .opc2 = 5, .access = PL1_R, .type = ARM_CP_CONST,
+ { .name = "ID_ISAR5", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 2, .opc2 = 5,
+ .access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_isar5 },
/* 6..7 are as yet unallocated and must RAZ */
{ .name = "ID_ISAR6", .cp = 15, .crn = 0, .crm = 2,
@@ -2014,7 +2247,12 @@ void register_cp_regs_for_features(ARMCPU *cpu)
{ .name = "ID_AA64DFR0_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 0,
.access = PL1_R, .type = ARM_CP_CONST,
- .resetvalue = cpu->id_aa64dfr0 },
+ /* We mask out the PMUVer field, beacuse we don't currently
+ * implement the PMU. Not advertising it prevents the guest
+ * from trying to use it and getting UNDEFs on registers we
+ * don't implement.
+ */
+ .resetvalue = cpu->id_aa64dfr0 & ~0xf00 },
{ .name = "ID_AA64DFR1_EL1", .state = ARM_CP_STATE_AA64,
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 5, .opc2 = 1,
.access = PL1_R, .type = ARM_CP_CONST,
@@ -2043,8 +2281,26 @@ void register_cp_regs_for_features(ARMCPU *cpu)
.opc0 = 3, .opc1 = 0, .crn = 0, .crm = 7, .opc2 = 1,
.access = PL1_R, .type = ARM_CP_CONST,
.resetvalue = cpu->id_aa64mmfr1 },
+ { .name = "MVFR0_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 0,
+ .access = PL1_R, .type = ARM_CP_CONST,
+ .resetvalue = cpu->mvfr0 },
+ { .name = "MVFR1_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 1,
+ .access = PL1_R, .type = ARM_CP_CONST,
+ .resetvalue = cpu->mvfr1 },
+ { .name = "MVFR2_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 3, .opc2 = 2,
+ .access = PL1_R, .type = ARM_CP_CONST,
+ .resetvalue = cpu->mvfr2 },
REGINFO_SENTINEL
};
+ ARMCPRegInfo rvbar = {
+ .name = "RVBAR_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 12, .crm = 0, .opc2 = 2,
+ .type = ARM_CP_CONST, .access = PL1_R, .resetvalue = cpu->rvbar
+ };
+ define_one_arm_cp_reg(cpu, &rvbar);
define_arm_cp_regs(cpu, v8_idregs);
define_arm_cp_regs(cpu, v8_cp_reginfo);
define_aarch64_debug_regs(cpu);
@@ -2098,8 +2354,9 @@ void register_cp_regs_for_features(ARMCPU *cpu)
* be read-only (ie write causes UNDEF exception).
*/
{
- ARMCPRegInfo id_cp_reginfo[] = {
- /* Note that the MIDR isn't a simple constant register because
+ ARMCPRegInfo id_pre_v8_midr_cp_reginfo[] = {
+ /* Pre-v8 MIDR space.
+ * Note that the MIDR isn't a simple constant register because
* of the TI925 behaviour where writes to another register can
* cause the MIDR value to change.
*
@@ -2113,22 +2370,6 @@ void register_cp_regs_for_features(ARMCPU *cpu)
.writefn = arm_cp_write_ignore, .raw_writefn = raw_write,
.fieldoffset = offsetof(CPUARMState, cp15.c0_cpuid),
.type = ARM_CP_OVERRIDE },
- { .name = "MIDR_EL1", .state = ARM_CP_STATE_AA64,
- .opc0 = 3, .opc1 = 0, .opc2 = 0, .crn = 0, .crm = 0,
- .access = PL1_R, .resetvalue = cpu->midr, .type = ARM_CP_CONST },
- { .name = "CTR",
- .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
- .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
- { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
- .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
- .access = PL0_R, .accessfn = ctr_el0_access,
- .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
- { .name = "TCMTR",
- .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
- .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
- { .name = "TLBTR",
- .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
- .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
/* crn = 0 op1 = 0 crm = 3..7 : currently unassigned; we RAZ. */
{ .name = "DUMMY",
.cp = 15, .crn = 0, .crm = 3, .opc1 = 0, .opc2 = CP_ANY,
@@ -2147,6 +2388,37 @@ void register_cp_regs_for_features(ARMCPU *cpu)
.access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
REGINFO_SENTINEL
};
+ ARMCPRegInfo id_v8_midr_cp_reginfo[] = {
+ /* v8 MIDR -- the wildcard isn't necessary, and nor is the
+ * variable-MIDR TI925 behaviour. Instead we have a single
+ * (strictly speaking IMPDEF) alias of the MIDR, REVIDR.
+ */
+ { .name = "MIDR_EL1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 0,
+ .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
+ { .name = "REVIDR_EL1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 0, .opc2 = 6,
+ .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->midr },
+ REGINFO_SENTINEL
+ };
+ ARMCPRegInfo id_cp_reginfo[] = {
+ /* These are common to v8 and pre-v8 */
+ { .name = "CTR",
+ .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 1,
+ .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
+ { .name = "CTR_EL0", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 3, .opc2 = 1, .crn = 0, .crm = 0,
+ .access = PL0_R, .accessfn = ctr_el0_access,
+ .type = ARM_CP_CONST, .resetvalue = cpu->ctr },
+ /* TCMTR and TLBTR exist in v8 but have no 64-bit versions */
+ { .name = "TCMTR",
+ .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 2,
+ .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+ { .name = "TLBTR",
+ .cp = 15, .crn = 0, .crm = 0, .opc1 = 0, .opc2 = 3,
+ .access = PL1_R, .type = ARM_CP_CONST, .resetvalue = 0 },
+ REGINFO_SENTINEL
+ };
ARMCPRegInfo crn0_wi_reginfo = {
.name = "CRN0_WI", .cp = 15, .crn = 0, .crm = CP_ANY,
.opc1 = CP_ANY, .opc2 = CP_ANY, .access = PL1_W,
@@ -2161,10 +2433,19 @@ void register_cp_regs_for_features(ARMCPU *cpu)
* UNDEF.
*/
define_one_arm_cp_reg(cpu, &crn0_wi_reginfo);
+ for (r = id_pre_v8_midr_cp_reginfo;
+ r->type != ARM_CP_SENTINEL; r++) {
+ r->access = PL1_RW;
+ }
for (r = id_cp_reginfo; r->type != ARM_CP_SENTINEL; r++) {
r->access = PL1_RW;
}
}
+ if (arm_feature(env, ARM_FEATURE_V8)) {
+ define_arm_cp_regs(cpu, id_v8_midr_cp_reginfo);
+ } else {
+ define_arm_cp_regs(cpu, id_pre_v8_midr_cp_reginfo);
+ }
define_arm_cp_regs(cpu, id_cp_reginfo);
}
@@ -2174,7 +2455,8 @@ void register_cp_regs_for_features(ARMCPU *cpu)
if (arm_feature(env, ARM_FEATURE_AUXCR)) {
ARMCPRegInfo auxcr = {
- .name = "AUXCR", .cp = 15, .crn = 1, .crm = 0, .opc1 = 0, .opc2 = 1,
+ .name = "ACTLR_EL1", .state = ARM_CP_STATE_BOTH,
+ .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 0, .opc2 = 1,
.access = PL1_RW, .type = ARM_CP_CONST,
.resetvalue = cpu->reset_auxcr
};
@@ -2182,12 +2464,39 @@ void register_cp_regs_for_features(ARMCPU *cpu)
}
if (arm_feature(env, ARM_FEATURE_CBAR)) {
- ARMCPRegInfo cbar = {
- .name = "CBAR", .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
- .access = PL1_R|PL3_W, .resetvalue = cpu->reset_cbar,
- .fieldoffset = offsetof(CPUARMState, cp15.c15_config_base_address)
- };
- define_one_arm_cp_reg(cpu, &cbar);
+ if (arm_feature(env, ARM_FEATURE_AARCH64)) {
+ /* 32 bit view is [31:18] 0...0 [43:32]. */
+ uint32_t cbar32 = (extract64(cpu->reset_cbar, 18, 14) << 18)
+ | extract64(cpu->reset_cbar, 32, 12);
+ ARMCPRegInfo cbar_reginfo[] = {
+ { .name = "CBAR",
+ .type = ARM_CP_CONST,
+ .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
+ .access = PL1_R, .resetvalue = cpu->reset_cbar },
+ { .name = "CBAR_EL1", .state = ARM_CP_STATE_AA64,
+ .type = ARM_CP_CONST,
+ .opc0 = 3, .opc1 = 1, .crn = 15, .crm = 3, .opc2 = 0,
+ .access = PL1_R, .resetvalue = cbar32 },
+ REGINFO_SENTINEL
+ };
+ /* We don't implement a r/w 64 bit CBAR currently */
+ assert(arm_feature(env, ARM_FEATURE_CBAR_RO));
+ define_arm_cp_regs(cpu, cbar_reginfo);
+ } else {
+ ARMCPRegInfo cbar = {
+ .name = "CBAR",
+ .cp = 15, .crn = 15, .crm = 0, .opc1 = 4, .opc2 = 0,
+ .access = PL1_R|PL3_W, .resetvalue = cpu->reset_cbar,
+ .fieldoffset = offsetof(CPUARMState,
+ cp15.c15_config_base_address)
+ };
+ if (arm_feature(env, ARM_FEATURE_CBAR_RO)) {
+ cbar.access = PL1_R;
+ cbar.fieldoffset = 0;
+ cbar.type = ARM_CP_CONST;
+ }
+ define_one_arm_cp_reg(cpu, &cbar);
+ }
}
/* Generic registers whose values depend on the implementation */
@@ -2684,12 +2993,11 @@ int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address, int rw,
ARMCPU *cpu = ARM_CPU(cs);
CPUARMState *env = &cpu->env;
+ env->exception.vaddress = address;
if (rw == 2) {
cs->exception_index = EXCP_PREFETCH_ABORT;
- env->cp15.c6_insn = address;
} else {
cs->exception_index = EXCP_DATA_ABORT;
- env->cp15.c6_data = address;
}
return 1;
}
@@ -2846,37 +3154,6 @@ static void do_v7m_exception_exit(CPUARMState *env)
pointer. */
}
-/* Exception names for debug logging; note that not all of these
- * precisely correspond to architectural exceptions.
- */
-static const char * const excnames[] = {
- [EXCP_UDEF] = "Undefined Instruction",
- [EXCP_SWI] = "SVC",
- [EXCP_PREFETCH_ABORT] = "Prefetch Abort",
- [EXCP_DATA_ABORT] = "Data Abort",
- [EXCP_IRQ] = "IRQ",
- [EXCP_FIQ] = "FIQ",
- [EXCP_BKPT] = "Breakpoint",
- [EXCP_EXCEPTION_EXIT] = "QEMU v7M exception exit",
- [EXCP_KERNEL_TRAP] = "QEMU intercept of kernel commpage",
- [EXCP_STREX] = "QEMU intercept of STREX",
-};
-
-static inline void arm_log_exception(int idx)
-{
- if (qemu_loglevel_mask(CPU_LOG_INT)) {
- const char *exc = NULL;
-
- if (idx >= 0 && idx < ARRAY_SIZE(excnames)) {
- exc = excnames[idx];
- }
- if (!exc) {
- exc = "unknown";
- }
- qemu_log_mask(CPU_LOG_INT, "Taking exception %d [%s]\n", idx, exc);
- }
-}
-
void arm_v7m_cpu_do_interrupt(CPUState *cs)
{
ARMCPU *cpu = ARM_CPU(cs);
@@ -2907,6 +3184,9 @@ void arm_v7m_cpu_do_interrupt(CPUState *cs)
return;
case EXCP_PREFETCH_ABORT:
case EXCP_DATA_ABORT:
+ /* TODO: if we implemented the MPU registers, this is where we
+ * should set the MMFAR, etc from exception.fsr and exception.vaddress.
+ */
armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_MEM);
return;
case EXCP_BKPT:
@@ -3021,19 +3301,26 @@ void arm_cpu_do_interrupt(CPUState *cs)
return;
}
}
- env->cp15.c5_insn = 2;
+ env->exception.fsr = 2;
/* Fall through to prefetch abort. */
case EXCP_PREFETCH_ABORT:
+ env->cp15.ifsr_el2 = env->exception.fsr;
+ env->cp15.far_el1 = deposit64(env->cp15.far_el1, 32, 32,
+ env->exception.vaddress);
qemu_log_mask(CPU_LOG_INT, "...with IFSR 0x%x IFAR 0x%x\n",
- env->cp15.c5_insn, env->cp15.c6_insn);
+ env->cp15.ifsr_el2, (uint32_t)env->exception.vaddress);
new_mode = ARM_CPU_MODE_ABT;
addr = 0x0c;
mask = CPSR_A | CPSR_I;
offset = 4;
break;
case EXCP_DATA_ABORT:
+ env->cp15.esr_el1 = env->exception.fsr;
+ env->cp15.far_el1 = deposit64(env->cp15.far_el1, 0, 32,
+ env->exception.vaddress);
qemu_log_mask(CPU_LOG_INT, "...with DFSR 0x%x DFAR 0x%x\n",
- env->cp15.c5_data, env->cp15.c6_data);
+ (uint32_t)env->cp15.esr_el1,
+ (uint32_t)env->exception.vaddress);
new_mode = ARM_CPU_MODE_ABT;
addr = 0x10;
mask = CPSR_A | CPSR_I;
@@ -3375,7 +3662,7 @@ typedef enum {
permission_fault = 3,
} MMUFaultType;
-static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
+static int get_phys_addr_lpae(CPUARMState *env, target_ulong address,
int access_type, int is_user,
hwaddr *phys_ptr, int *prot,
target_ulong *page_size_ptr)
@@ -3385,26 +3672,46 @@ static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
MMUFaultType fault_type = translation_fault;
uint32_t level = 1;
uint32_t epd;
- uint32_t tsz;
+ int32_t tsz;
+ uint32_t tg;
uint64_t ttbr;
int ttbr_select;
- int n;
- hwaddr descaddr;
+ hwaddr descaddr, descmask;
uint32_t tableattrs;
target_ulong page_size;
uint32_t attrs;
+ int32_t granule_sz = 9;
+ int32_t va_size = 32;
+ int32_t tbi = 0;
+
+ if (arm_el_is_aa64(env, 1)) {
+ va_size = 64;
+ if (extract64(address, 55, 1))
+ tbi = extract64(env->cp15.c2_control, 38, 1);
+ else
+ tbi = extract64(env->cp15.c2_control, 37, 1);
+ tbi *= 8;
+ }
/* Determine whether this address is in the region controlled by
* TTBR0 or TTBR1 (or if it is in neither region and should fault).
* This is a Non-secure PL0/1 stage 1 translation, so controlled by
* TTBCR/TTBR0/TTBR1 in accordance with ARM ARM DDI0406C table B-32:
*/
- uint32_t t0sz = extract32(env->cp15.c2_control, 0, 3);
- uint32_t t1sz = extract32(env->cp15.c2_control, 16, 3);
- if (t0sz && !extract32(address, 32 - t0sz, t0sz)) {
+ uint32_t t0sz = extract32(env->cp15.c2_control, 0, 6);
+ if (arm_el_is_aa64(env, 1)) {
+ t0sz = MIN(t0sz, 39);
+ t0sz = MAX(t0sz, 16);
+ }
+ uint32_t t1sz = extract32(env->cp15.c2_control, 16, 6);
+ if (arm_el_is_aa64(env, 1)) {
+ t1sz = MIN(t1sz, 39);
+ t1sz = MAX(t1sz, 16);
+ }
+ if (t0sz && !extract64(address, va_size - t0sz, t0sz - tbi)) {
/* there is a ttbr0 region and we are in it (high bits all zero) */
ttbr_select = 0;
- } else if (t1sz && !extract32(~address, 32 - t1sz, t1sz)) {
+ } else if (t1sz && !extract64(~address, va_size - t1sz, t1sz - tbi)) {
/* there is a ttbr1 region and we are in it (high bits all one) */
ttbr_select = 1;
} else if (!t0sz) {
@@ -3430,10 +3737,26 @@ static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
ttbr = env->cp15.ttbr0_el1;
epd = extract32(env->cp15.c2_control, 7, 1);
tsz = t0sz;
+
+ tg = extract32(env->cp15.c2_control, 14, 2);
+ if (tg == 1) { /* 64KB pages */
+ granule_sz = 13;
+ }
+ if (tg == 2) { /* 16KB pages */
+ granule_sz = 11;
+ }
} else {
ttbr = env->cp15.ttbr1_el1;
epd = extract32(env->cp15.c2_control, 23, 1);
tsz = t1sz;
+
+ tg = extract32(env->cp15.c2_control, 30, 2);
+ if (tg == 3) { /* 64KB pages */
+ granule_sz = 13;
+ }
+ if (tg == 1) { /* 16KB pages */
+ granule_sz = 11;
+ }
}
if (epd) {
@@ -3441,34 +3764,37 @@ static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
goto do_fault;
}
- /* If the region is small enough we will skip straight to a 2nd level
- * lookup. This affects the number of bits of the address used in
- * combination with the TTBR to find the first descriptor. ('n' here
- * matches the usage in the ARM ARM sB3.6.6, where bits [39..n] are
- * from the TTBR, [n-1..3] from the vaddr, and [2..0] always zero).
+ /* The starting level depends on the virtual address size which can be
+ * up to 48-bits and the translation granule size.
*/
- if (tsz > 1) {
- level = 2;
- n = 14 - tsz;
+ if ((va_size - tsz) > (granule_sz * 4 + 3)) {
+ level = 0;
+ } else if ((va_size - tsz) > (granule_sz * 3 + 3)) {
+ level = 1;
} else {
- n = 5 - tsz;
+ level = 2;
}
/* Clear the vaddr bits which aren't part of the within-region address,
* so that we don't have to special case things when calculating the
* first descriptor address.
*/
- address &= (0xffffffffU >> tsz);
+ if (tsz) {
+ address &= (1ULL << (va_size - tsz)) - 1;
+ }
+
+ descmask = (1ULL << (granule_sz + 3)) - 1;
/* Now we can extract the actual base address from the TTBR */
- descaddr = extract64(ttbr, 0, 40);
- descaddr &= ~((1ULL << n) - 1);
+ descaddr = extract64(ttbr, 0, 48);
+ descaddr &= ~((1ULL << (va_size - tsz - (granule_sz * (4 - level)))) - 1);
tableattrs = 0;
for (;;) {
uint64_t descriptor;
- descaddr |= ((address >> (9 * (4 - level))) & 0xff8);
+ descaddr |= (address >> (granule_sz * (4 - level))) & descmask;
+ descaddr &= ~7ULL;
descriptor = ldq_phys(cs->as, descaddr);
if (!(descriptor & 1) ||
(!(descriptor & 2) && (level == 3))) {
@@ -3491,11 +3817,16 @@ static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
* These are basically the same thing, although the number
* of bits we pull in from the vaddr varies.
*/
- page_size = (1 << (39 - (9 * level)));
+ page_size = (1 << ((granule_sz * (4 - level)) + 3));
descaddr |= (address & (page_size - 1));
/* Extract attributes from the descriptor and merge with table attrs */
- attrs = extract64(descriptor, 2, 10)
- | (extract64(descriptor, 52, 12) << 10);
+ if (arm_feature(env, ARM_FEATURE_V8)) {
+ attrs = extract64(descriptor, 2, 10)
+ | (extract64(descriptor, 53, 11) << 10);
+ } else {
+ attrs = extract64(descriptor, 2, 10)
+ | (extract64(descriptor, 52, 12) << 10);
+ }
attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */
attrs |= extract32(tableattrs, 3, 1) << 5; /* APTable[1] => AP[2] */
/* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1
@@ -3569,9 +3900,9 @@ static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
return 2;
if (access_type == 2) {
- mask = env->cp15.c5_insn;
+ mask = env->cp15.pmsav5_insn_ap;
} else {
- mask = env->cp15.c5_data;
+ mask = env->cp15.pmsav5_data_ap;
}
mask = (mask >> (n * 4)) & 0xf;
switch (mask) {
@@ -3629,7 +3960,7 @@ static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
* @prot: set to the permissions for the page containing phys_ptr
* @page_size: set to the size of the page containing phys_ptr
*/
-static inline int get_phys_addr(CPUARMState *env, uint32_t address,
+static inline int get_phys_addr(CPUARMState *env, target_ulong address,
int access_type, int is_user,
hwaddr *phys_ptr, int *prot,
target_ulong *page_size)
@@ -3669,6 +4000,8 @@ int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
target_ulong page_size;
int prot;
int ret, is_user;
+ uint32_t syn;
+ bool same_el = (arm_current_pl(env) != 0);
is_user = mmu_idx == MMU_USER_IDX;
ret = get_phys_addr(env, address, access_type, is_user, &phys_addr, &prot,
@@ -3676,22 +4009,31 @@ int arm_cpu_handle_mmu_fault(CPUState *cs, vaddr address,
if (ret == 0) {
/* Map a single [sub]page. */
phys_addr &= ~(hwaddr)0x3ff;
- address &= ~(uint32_t)0x3ff;
+ address &= ~(target_ulong)0x3ff;
tlb_set_page(cs, address, phys_addr, prot, mmu_idx, page_size);
return 0;
}
+ /* AArch64 syndrome does not have an LPAE bit */
+ syn = ret & ~(1 << 9);
+
+ /* For insn and data aborts we assume there is no instruction syndrome
+ * information; this is always true for exceptions reported to EL1.
+ */
if (access_type == 2) {
- env->cp15.c5_insn = ret;
- env->cp15.c6_insn = address;
+ syn = syn_insn_abort(same_el, 0, 0, syn);
cs->exception_index = EXCP_PREFETCH_ABORT;
} else {
- env->cp15.c5_data = ret;
- if (access_type == 1 && arm_feature(env, ARM_FEATURE_V6))
- env->cp15.c5_data |= (1 << 11);
- env->cp15.c6_data = address;
+ syn = syn_data_abort(same_el, 0, 0, 0, access_type == 1, syn);
+ if (access_type == 1 && arm_feature(env, ARM_FEATURE_V6)) {
+ ret |= (1 << 11);
+ }
cs->exception_index = EXCP_DATA_ABORT;
}
+
+ env->exception.syndrome = syn;
+ env->exception.vaddress = address;
+ env->exception.fsr = ret;
return 1;
}
@@ -3842,6 +4184,88 @@ void HELPER(v7m_msr)(CPUARMState *env, uint32_t reg, uint32_t val)
#endif
+void HELPER(dc_zva)(CPUARMState *env, uint64_t vaddr_in)
+{
+ /* Implement DC ZVA, which zeroes a fixed-length block of memory.
+ * Note that we do not implement the (architecturally mandated)
+ * alignment fault for attempts to use this on Device memory
+ * (which matches the usual QEMU behaviour of not implementing either
+ * alignment faults or any memory attribute handling).
+ */
+
+ ARMCPU *cpu = arm_env_get_cpu(env);
+ uint64_t blocklen = 4 << cpu->dcz_blocksize;
+ uint64_t vaddr = vaddr_in & ~(blocklen - 1);
+
+#ifndef CONFIG_USER_ONLY
+ {
+ /* Slightly awkwardly, QEMU's TARGET_PAGE_SIZE may be less than
+ * the block size so we might have to do more than one TLB lookup.
+ * We know that in fact for any v8 CPU the page size is at least 4K
+ * and the block size must be 2K or less, but TARGET_PAGE_SIZE is only
+ * 1K as an artefact of legacy v5 subpage support being present in the
+ * same QEMU executable.
+ */
+ int maxidx = DIV_ROUND_UP(blocklen, TARGET_PAGE_SIZE);
+ void *hostaddr[maxidx];
+ int try, i;
+
+ for (try = 0; try < 2; try++) {
+
+ for (i = 0; i < maxidx; i++) {
+ hostaddr[i] = tlb_vaddr_to_host(env,
+ vaddr + TARGET_PAGE_SIZE * i,
+ 1, cpu_mmu_index(env));
+ if (!hostaddr[i]) {
+ break;
+ }
+ }
+ if (i == maxidx) {
+ /* If it's all in the TLB it's fair game for just writing to;
+ * we know we don't need to update dirty status, etc.
+ */
+ for (i = 0; i < maxidx - 1; i++) {
+ memset(hostaddr[i], 0, TARGET_PAGE_SIZE);
+ }
+ memset(hostaddr[i], 0, blocklen - (i * TARGET_PAGE_SIZE));
+ return;
+ }
+ /* OK, try a store and see if we can populate the tlb. This
+ * might cause an exception if the memory isn't writable,
+ * in which case we will longjmp out of here. We must for
+ * this purpose use the actual register value passed to us
+ * so that we get the fault address right.
+ */
+ helper_ret_stb_mmu(env, vaddr_in, 0, cpu_mmu_index(env), GETRA());
+ /* Now we can populate the other TLB entries, if any */
+ for (i = 0; i < maxidx; i++) {
+ uint64_t va = vaddr + TARGET_PAGE_SIZE * i;
+ if (va != (vaddr_in & TARGET_PAGE_MASK)) {
+ helper_ret_stb_mmu(env, va, 0, cpu_mmu_index(env), GETRA());
+ }
+ }
+ }
+
+ /* Slow path (probably attempt to do this to an I/O device or
+ * similar, or clearing of a block of code we have translations
+ * cached for). Just do a series of byte writes as the architecture
+ * demands. It's not worth trying to use a cpu_physical_memory_map(),
+ * memset(), unmap() sequence here because:
+ * + we'd need to account for the blocksize being larger than a page
+ * + the direct-RAM access case is almost always going to be dealt
+ * with in the fastpath code above, so there's no speed benefit
+ * + we would have to deal with the map returning NULL because the
+ * bounce buffer was in use
+ */
+ for (i = 0; i < blocklen; i++) {
+ helper_ret_stb_mmu(env, vaddr + i, 0, cpu_mmu_index(env), GETRA());
+ }
+ }
+#else
+ memset(g2h(vaddr), 0, blocklen);
+#endif
+}
+
/* Note that signed overflow is undefined in C. The following routines are
careful to use unsigned types where modulo arithmetic is required.
Failure to do so _will_ break on newer gcc. */