aboutsummaryrefslogtreecommitdiff
path: root/target
diff options
context:
space:
mode:
Diffstat (limited to 'target')
-rw-r--r--target/arm/cpu.h120
-rw-r--r--target/arm/cpu64.c4
-rw-r--r--target/arm/crypto_helper.c277
-rw-r--r--target/arm/helper.c548
-rw-r--r--target/arm/helper.h12
-rw-r--r--target/arm/kvm_arm.h4
-rw-r--r--target/arm/machine.c88
-rw-r--r--target/arm/translate-a64.c1325
-rw-r--r--target/arm/translate.c8
-rw-r--r--target/arm/translate.h2
-rw-r--r--target/s390x/cpu.c10
-rw-r--r--target/s390x/cpu.h99
-rw-r--r--target/s390x/cpu_features.c5
-rw-r--r--target/s390x/cpu_features_def.h4
-rw-r--r--target/s390x/cpu_models.c6
-rw-r--r--target/s390x/excp_helper.c147
-rw-r--r--target/s390x/gen-features.c17
-rw-r--r--target/s390x/helper.h10
-rw-r--r--target/s390x/insn-data.def14
-rw-r--r--target/s390x/internal.h5
-rw-r--r--target/s390x/interrupt.c100
-rw-r--r--target/s390x/kvm-stub.c13
-rw-r--r--target/s390x/kvm.c78
-rw-r--r--target/s390x/kvm_s390x.h10
-rw-r--r--target/s390x/misc_helper.c355
-rw-r--r--target/s390x/translate.c110
26 files changed, 2580 insertions, 791 deletions
diff --git a/target/arm/cpu.h b/target/arm/cpu.h
index d2bb59eded..521444a5a1 100644
--- a/target/arm/cpu.h
+++ b/target/arm/cpu.h
@@ -153,6 +153,49 @@ typedef struct {
uint32_t base_mask;
} TCR;
+/* Define a maximum sized vector register.
+ * For 32-bit, this is a 128-bit NEON/AdvSIMD register.
+ * For 64-bit, this is a 2048-bit SVE register.
+ *
+ * Note that the mapping between S, D, and Q views of the register bank
+ * differs between AArch64 and AArch32.
+ * In AArch32:
+ * Qn = regs[n].d[1]:regs[n].d[0]
+ * Dn = regs[n / 2].d[n & 1]
+ * Sn = regs[n / 4].d[n % 4 / 2],
+ * bits 31..0 for even n, and bits 63..32 for odd n
+ * (and regs[16] to regs[31] are inaccessible)
+ * In AArch64:
+ * Zn = regs[n].d[*]
+ * Qn = regs[n].d[1]:regs[n].d[0]
+ * Dn = regs[n].d[0]
+ * Sn = regs[n].d[0] bits 31..0
+ *
+ * This corresponds to the architecturally defined mapping between
+ * the two execution states, and means we do not need to explicitly
+ * map these registers when changing states.
+ *
+ * Align the data for use with TCG host vector operations.
+ */
+
+#ifdef TARGET_AARCH64
+# define ARM_MAX_VQ 16
+#else
+# define ARM_MAX_VQ 1
+#endif
+
+typedef struct ARMVectorReg {
+ uint64_t d[2 * ARM_MAX_VQ] QEMU_ALIGNED(16);
+} ARMVectorReg;
+
+/* In AArch32 mode, predicate registers do not exist at all. */
+#ifdef TARGET_AARCH64
+typedef struct ARMPredicateReg {
+ uint64_t p[2 * ARM_MAX_VQ / 8] QEMU_ALIGNED(16);
+} ARMPredicateReg;
+#endif
+
+
typedef struct CPUARMState {
/* Regs for current mode. */
uint32_t regs[16];
@@ -477,22 +520,12 @@ typedef struct CPUARMState {
/* VFP coprocessor state. */
struct {
- /* VFP/Neon register state. Note that the mapping between S, D and Q
- * views of the register bank differs between AArch64 and AArch32:
- * In AArch32:
- * Qn = regs[2n+1]:regs[2n]
- * Dn = regs[n]
- * Sn = regs[n/2] bits 31..0 for even n, and bits 63..32 for odd n
- * (and regs[32] to regs[63] are inaccessible)
- * In AArch64:
- * Qn = regs[2n+1]:regs[2n]
- * Dn = regs[2n]
- * Sn = regs[2n] bits 31..0
- * This corresponds to the architecturally defined mapping between
- * the two execution states, and means we do not need to explicitly
- * map these registers when changing states.
- */
- uint64_t regs[64];
+ ARMVectorReg zregs[32];
+
+#ifdef TARGET_AARCH64
+ /* Store FFR as pregs[16] to make it easier to treat as any other. */
+ ARMPredicateReg pregs[17];
+#endif
uint32_t xregs[16];
/* We store these fpcsr fields separately for convenience. */
@@ -516,6 +549,9 @@ typedef struct CPUARMState {
*/
float_status fp_status;
float_status standard_fp_status;
+
+ /* ZCR_EL[1-3] */
+ uint64_t zcr_el[4];
} vfp;
uint64_t exclusive_addr;
uint64_t exclusive_val;
@@ -890,6 +926,8 @@ void pmccntr_sync(CPUARMState *env);
#define CPTR_TCPAC (1U << 31)
#define CPTR_TTA (1U << 20)
#define CPTR_TFP (1U << 10)
+#define CPTR_TZ (1U << 8) /* CPTR_EL2 */
+#define CPTR_EZ (1U << 8) /* CPTR_EL3 */
#define MDCR_EPMAD (1U << 21)
#define MDCR_EDAD (1U << 20)
@@ -1341,6 +1379,10 @@ enum arm_features {
ARM_FEATURE_M_SECURITY, /* M profile Security Extension */
ARM_FEATURE_JAZELLE, /* has (trivial) Jazelle implementation */
ARM_FEATURE_SVE, /* has Scalable Vector Extension */
+ ARM_FEATURE_V8_SHA512, /* implements SHA512 part of v8 Crypto Extensions */
+ ARM_FEATURE_V8_SHA3, /* implements SHA3 part of v8 Crypto Extensions */
+ ARM_FEATURE_V8_SM3, /* implements SM3 part of v8 Crypto Extensions */
+ ARM_FEATURE_V8_SM4, /* implements SM4 part of v8 Crypto Extensions */
};
static inline int arm_feature(CPUARMState *env, int feature)
@@ -1506,16 +1548,42 @@ static inline bool armv7m_nvic_can_take_pending_exception(void *opaque)
*/
void armv7m_nvic_set_pending(void *opaque, int irq, bool secure);
/**
+ * armv7m_nvic_set_pending_derived: mark this derived exception as pending
+ * @opaque: the NVIC
+ * @irq: the exception number to mark pending
+ * @secure: false for non-banked exceptions or for the nonsecure
+ * version of a banked exception, true for the secure version of a banked
+ * exception.
+ *
+ * Similar to armv7m_nvic_set_pending(), but specifically for derived
+ * exceptions (exceptions generated in the course of trying to take
+ * a different exception).
+ */
+void armv7m_nvic_set_pending_derived(void *opaque, int irq, bool secure);
+/**
+ * armv7m_nvic_get_pending_irq_info: return highest priority pending
+ * exception, and whether it targets Secure state
+ * @opaque: the NVIC
+ * @pirq: set to pending exception number
+ * @ptargets_secure: set to whether pending exception targets Secure
+ *
+ * This function writes the number of the highest priority pending
+ * exception (the one which would be made active by
+ * armv7m_nvic_acknowledge_irq()) to @pirq, and sets @ptargets_secure
+ * to true if the current highest priority pending exception should
+ * be taken to Secure state, false for NS.
+ */
+void armv7m_nvic_get_pending_irq_info(void *opaque, int *pirq,
+ bool *ptargets_secure);
+/**
* armv7m_nvic_acknowledge_irq: make highest priority pending exception active
* @opaque: the NVIC
*
* Move the current highest priority pending exception from the pending
* state to the active state, and update v7m.exception to indicate that
* it is the exception currently being handled.
- *
- * Returns: true if exception should be taken to Secure state, false for NS
*/
-bool armv7m_nvic_acknowledge_irq(void *opaque);
+void armv7m_nvic_acknowledge_irq(void *opaque);
/**
* armv7m_nvic_complete_irq: complete specified interrupt or exception
* @opaque: the NVIC
@@ -2610,6 +2678,10 @@ static inline bool arm_cpu_data_is_big_endian(CPUARMState *env)
#define ARM_TBFLAG_TBI0_MASK (0x1ull << ARM_TBFLAG_TBI0_SHIFT)
#define ARM_TBFLAG_TBI1_SHIFT 1 /* TBI1 for EL0/1 */
#define ARM_TBFLAG_TBI1_MASK (0x1ull << ARM_TBFLAG_TBI1_SHIFT)
+#define ARM_TBFLAG_SVEEXC_EL_SHIFT 2
+#define ARM_TBFLAG_SVEEXC_EL_MASK (0x3 << ARM_TBFLAG_SVEEXC_EL_SHIFT)
+#define ARM_TBFLAG_ZCR_LEN_SHIFT 4
+#define ARM_TBFLAG_ZCR_LEN_MASK (0xf << ARM_TBFLAG_ZCR_LEN_SHIFT)
/* some convenience accessor macros */
#define ARM_TBFLAG_AARCH64_STATE(F) \
@@ -2646,6 +2718,10 @@ static inline bool arm_cpu_data_is_big_endian(CPUARMState *env)
(((F) & ARM_TBFLAG_TBI0_MASK) >> ARM_TBFLAG_TBI0_SHIFT)
#define ARM_TBFLAG_TBI1(F) \
(((F) & ARM_TBFLAG_TBI1_MASK) >> ARM_TBFLAG_TBI1_SHIFT)
+#define ARM_TBFLAG_SVEEXC_EL(F) \
+ (((F) & ARM_TBFLAG_SVEEXC_EL_MASK) >> ARM_TBFLAG_SVEEXC_EL_SHIFT)
+#define ARM_TBFLAG_ZCR_LEN(F) \
+ (((F) & ARM_TBFLAG_ZCR_LEN_MASK) >> ARM_TBFLAG_ZCR_LEN_SHIFT)
static inline bool bswap_code(bool sctlr_b)
{
@@ -2769,7 +2845,7 @@ static inline void *arm_get_el_change_hook_opaque(ARMCPU *cpu)
*/
static inline uint64_t *aa32_vfp_dreg(CPUARMState *env, unsigned regno)
{
- return &env->vfp.regs[regno];
+ return &env->vfp.zregs[regno >> 1].d[regno & 1];
}
/**
@@ -2778,7 +2854,7 @@ static inline uint64_t *aa32_vfp_dreg(CPUARMState *env, unsigned regno)
*/
static inline uint64_t *aa32_vfp_qreg(CPUARMState *env, unsigned regno)
{
- return &env->vfp.regs[2 * regno];
+ return &env->vfp.zregs[regno].d[0];
}
/**
@@ -2787,7 +2863,7 @@ static inline uint64_t *aa32_vfp_qreg(CPUARMState *env, unsigned regno)
*/
static inline uint64_t *aa64_vfp_qreg(CPUARMState *env, unsigned regno)
{
- return &env->vfp.regs[2 * regno];
+ return &env->vfp.zregs[regno].d[0];
}
#endif
diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c
index 670c07ab6e..1c330adc28 100644
--- a/target/arm/cpu64.c
+++ b/target/arm/cpu64.c
@@ -224,6 +224,10 @@ static void aarch64_any_initfn(Object *obj)
set_feature(&cpu->env, ARM_FEATURE_V8_AES);
set_feature(&cpu->env, ARM_FEATURE_V8_SHA1);
set_feature(&cpu->env, ARM_FEATURE_V8_SHA256);
+ set_feature(&cpu->env, ARM_FEATURE_V8_SHA512);
+ set_feature(&cpu->env, ARM_FEATURE_V8_SHA3);
+ set_feature(&cpu->env, ARM_FEATURE_V8_SM3);
+ set_feature(&cpu->env, ARM_FEATURE_V8_SM4);
set_feature(&cpu->env, ARM_FEATURE_V8_PMULL);
set_feature(&cpu->env, ARM_FEATURE_CRC);
cpu->ctr = 0x80038003; /* 32 byte I and D cacheline size, VIPT icache */
diff --git a/target/arm/crypto_helper.c b/target/arm/crypto_helper.c
index 9ca0bdead7..cc339ea7e0 100644
--- a/target/arm/crypto_helper.c
+++ b/target/arm/crypto_helper.c
@@ -1,7 +1,7 @@
/*
* crypto_helper.c - emulate v8 Crypto Extensions instructions
*
- * Copyright (C) 2013 - 2014 Linaro Ltd <ard.biesheuvel@linaro.org>
+ * Copyright (C) 2013 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
@@ -419,3 +419,278 @@ void HELPER(crypto_sha256su1)(void *vd, void *vn, void *vm)
rd[0] = d.l[0];
rd[1] = d.l[1];
}
+
+/*
+ * The SHA-512 logical functions (same as above but using 64-bit operands)
+ */
+
+static uint64_t cho512(uint64_t x, uint64_t y, uint64_t z)
+{
+ return (x & (y ^ z)) ^ z;
+}
+
+static uint64_t maj512(uint64_t x, uint64_t y, uint64_t z)
+{
+ return (x & y) | ((x | y) & z);
+}
+
+static uint64_t S0_512(uint64_t x)
+{
+ return ror64(x, 28) ^ ror64(x, 34) ^ ror64(x, 39);
+}
+
+static uint64_t S1_512(uint64_t x)
+{
+ return ror64(x, 14) ^ ror64(x, 18) ^ ror64(x, 41);
+}
+
+static uint64_t s0_512(uint64_t x)
+{
+ return ror64(x, 1) ^ ror64(x, 8) ^ (x >> 7);
+}
+
+static uint64_t s1_512(uint64_t x)
+{
+ return ror64(x, 19) ^ ror64(x, 61) ^ (x >> 6);
+}
+
+void HELPER(crypto_sha512h)(void *vd, void *vn, void *vm)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ uint64_t *rm = vm;
+ uint64_t d0 = rd[0];
+ uint64_t d1 = rd[1];
+
+ d1 += S1_512(rm[1]) + cho512(rm[1], rn[0], rn[1]);
+ d0 += S1_512(d1 + rm[0]) + cho512(d1 + rm[0], rm[1], rn[0]);
+
+ rd[0] = d0;
+ rd[1] = d1;
+}
+
+void HELPER(crypto_sha512h2)(void *vd, void *vn, void *vm)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ uint64_t *rm = vm;
+ uint64_t d0 = rd[0];
+ uint64_t d1 = rd[1];
+
+ d1 += S0_512(rm[0]) + maj512(rn[0], rm[1], rm[0]);
+ d0 += S0_512(d1) + maj512(d1, rm[0], rm[1]);
+
+ rd[0] = d0;
+ rd[1] = d1;
+}
+
+void HELPER(crypto_sha512su0)(void *vd, void *vn)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ uint64_t d0 = rd[0];
+ uint64_t d1 = rd[1];
+
+ d0 += s0_512(rd[1]);
+ d1 += s0_512(rn[0]);
+
+ rd[0] = d0;
+ rd[1] = d1;
+}
+
+void HELPER(crypto_sha512su1)(void *vd, void *vn, void *vm)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ uint64_t *rm = vm;
+
+ rd[0] += s1_512(rn[0]) + rm[0];
+ rd[1] += s1_512(rn[1]) + rm[1];
+}
+
+void HELPER(crypto_sm3partw1)(void *vd, void *vn, void *vm)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ uint64_t *rm = vm;
+ union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
+ union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
+ union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
+ uint32_t t;
+
+ t = CR_ST_WORD(d, 0) ^ CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 1), 17);
+ CR_ST_WORD(d, 0) = t ^ ror32(t, 17) ^ ror32(t, 9);
+
+ t = CR_ST_WORD(d, 1) ^ CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 2), 17);
+ CR_ST_WORD(d, 1) = t ^ ror32(t, 17) ^ ror32(t, 9);
+
+ t = CR_ST_WORD(d, 2) ^ CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 3), 17);
+ CR_ST_WORD(d, 2) = t ^ ror32(t, 17) ^ ror32(t, 9);
+
+ t = CR_ST_WORD(d, 3) ^ CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 0), 17);
+ CR_ST_WORD(d, 3) = t ^ ror32(t, 17) ^ ror32(t, 9);
+
+ rd[0] = d.l[0];
+ rd[1] = d.l[1];
+}
+
+void HELPER(crypto_sm3partw2)(void *vd, void *vn, void *vm)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ uint64_t *rm = vm;
+ union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
+ union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
+ union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
+ uint32_t t = CR_ST_WORD(n, 0) ^ ror32(CR_ST_WORD(m, 0), 25);
+
+ CR_ST_WORD(d, 0) ^= t;
+ CR_ST_WORD(d, 1) ^= CR_ST_WORD(n, 1) ^ ror32(CR_ST_WORD(m, 1), 25);
+ CR_ST_WORD(d, 2) ^= CR_ST_WORD(n, 2) ^ ror32(CR_ST_WORD(m, 2), 25);
+ CR_ST_WORD(d, 3) ^= CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(m, 3), 25) ^
+ ror32(t, 17) ^ ror32(t, 2) ^ ror32(t, 26);
+
+ rd[0] = d.l[0];
+ rd[1] = d.l[1];
+}
+
+void HELPER(crypto_sm3tt)(void *vd, void *vn, void *vm, uint32_t imm2,
+ uint32_t opcode)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ uint64_t *rm = vm;
+ union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
+ union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
+ union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
+ uint32_t t;
+
+ assert(imm2 < 4);
+
+ if (opcode == 0 || opcode == 2) {
+ /* SM3TT1A, SM3TT2A */
+ t = par(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
+ } else if (opcode == 1) {
+ /* SM3TT1B */
+ t = maj(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
+ } else if (opcode == 3) {
+ /* SM3TT2B */
+ t = cho(CR_ST_WORD(d, 3), CR_ST_WORD(d, 2), CR_ST_WORD(d, 1));
+ } else {
+ g_assert_not_reached();
+ }
+
+ t += CR_ST_WORD(d, 0) + CR_ST_WORD(m, imm2);
+
+ CR_ST_WORD(d, 0) = CR_ST_WORD(d, 1);
+
+ if (opcode < 2) {
+ /* SM3TT1A, SM3TT1B */
+ t += CR_ST_WORD(n, 3) ^ ror32(CR_ST_WORD(d, 3), 20);
+
+ CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 23);
+ } else {
+ /* SM3TT2A, SM3TT2B */
+ t += CR_ST_WORD(n, 3);
+ t ^= rol32(t, 9) ^ rol32(t, 17);
+
+ CR_ST_WORD(d, 1) = ror32(CR_ST_WORD(d, 2), 13);
+ }
+
+ CR_ST_WORD(d, 2) = CR_ST_WORD(d, 3);
+ CR_ST_WORD(d, 3) = t;
+
+ rd[0] = d.l[0];
+ rd[1] = d.l[1];
+}
+
+static uint8_t const sm4_sbox[] = {
+ 0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7,
+ 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05,
+ 0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3,
+ 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99,
+ 0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a,
+ 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62,
+ 0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95,
+ 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6,
+ 0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba,
+ 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8,
+ 0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b,
+ 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35,
+ 0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2,
+ 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87,
+ 0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52,
+ 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e,
+ 0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5,
+ 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1,
+ 0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55,
+ 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3,
+ 0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60,
+ 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f,
+ 0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f,
+ 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51,
+ 0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f,
+ 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8,
+ 0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd,
+ 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0,
+ 0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e,
+ 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84,
+ 0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20,
+ 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48,
+};
+
+void HELPER(crypto_sm4e)(void *vd, void *vn)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ union CRYPTO_STATE d = { .l = { rd[0], rd[1] } };
+ union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
+ uint32_t t, i;
+
+ for (i = 0; i < 4; i++) {
+ t = CR_ST_WORD(d, (i + 1) % 4) ^
+ CR_ST_WORD(d, (i + 2) % 4) ^
+ CR_ST_WORD(d, (i + 3) % 4) ^
+ CR_ST_WORD(n, i);
+
+ t = sm4_sbox[t & 0xff] |
+ sm4_sbox[(t >> 8) & 0xff] << 8 |
+ sm4_sbox[(t >> 16) & 0xff] << 16 |
+ sm4_sbox[(t >> 24) & 0xff] << 24;
+
+ CR_ST_WORD(d, i) ^= t ^ rol32(t, 2) ^ rol32(t, 10) ^ rol32(t, 18) ^
+ rol32(t, 24);
+ }
+
+ rd[0] = d.l[0];
+ rd[1] = d.l[1];
+}
+
+void HELPER(crypto_sm4ekey)(void *vd, void *vn, void* vm)
+{
+ uint64_t *rd = vd;
+ uint64_t *rn = vn;
+ uint64_t *rm = vm;
+ union CRYPTO_STATE d;
+ union CRYPTO_STATE n = { .l = { rn[0], rn[1] } };
+ union CRYPTO_STATE m = { .l = { rm[0], rm[1] } };
+ uint32_t t, i;
+
+ d = n;
+ for (i = 0; i < 4; i++) {
+ t = CR_ST_WORD(d, (i + 1) % 4) ^
+ CR_ST_WORD(d, (i + 2) % 4) ^
+ CR_ST_WORD(d, (i + 3) % 4) ^
+ CR_ST_WORD(m, i);
+
+ t = sm4_sbox[t & 0xff] |
+ sm4_sbox[(t >> 8) & 0xff] << 8 |
+ sm4_sbox[(t >> 16) & 0xff] << 16 |
+ sm4_sbox[(t >> 24) & 0xff] << 24;
+
+ CR_ST_WORD(d, i) ^= t ^ rol32(t, 13) ^ rol32(t, 23);
+ }
+
+ rd[0] = d.l[0];
+ rd[1] = d.l[1];
+}
diff --git a/target/arm/helper.c b/target/arm/helper.c
index bfce09643b..180ab75458 100644
--- a/target/arm/helper.c
+++ b/target/arm/helper.c
@@ -4266,6 +4266,125 @@ static const ARMCPRegInfo debug_lpae_cp_reginfo[] = {
REGINFO_SENTINEL
};
+/* Return the exception level to which SVE-disabled exceptions should
+ * be taken, or 0 if SVE is enabled.
+ */
+static int sve_exception_el(CPUARMState *env)
+{
+#ifndef CONFIG_USER_ONLY
+ unsigned current_el = arm_current_el(env);
+
+ /* The CPACR.ZEN controls traps to EL1:
+ * 0, 2 : trap EL0 and EL1 accesses
+ * 1 : trap only EL0 accesses
+ * 3 : trap no accesses
+ */
+ switch (extract32(env->cp15.cpacr_el1, 16, 2)) {
+ default:
+ if (current_el <= 1) {
+ /* Trap to PL1, which might be EL1 or EL3 */
+ if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) {
+ return 3;
+ }
+ return 1;
+ }
+ break;
+ case 1:
+ if (current_el == 0) {
+ return 1;
+ }
+ break;
+ case 3:
+ break;
+ }
+
+ /* Similarly for CPACR.FPEN, after having checked ZEN. */
+ switch (extract32(env->cp15.cpacr_el1, 20, 2)) {
+ default:
+ if (current_el <= 1) {
+ if (arm_is_secure(env) && !arm_el_is_aa64(env, 3)) {
+ return 3;
+ }
+ return 1;
+ }
+ break;
+ case 1:
+ if (current_el == 0) {
+ return 1;
+ }
+ break;
+ case 3:
+ break;
+ }
+
+ /* CPTR_EL2. Check both TZ and TFP. */
+ if (current_el <= 2
+ && (env->cp15.cptr_el[2] & (CPTR_TFP | CPTR_TZ))
+ && !arm_is_secure_below_el3(env)) {
+ return 2;
+ }
+
+ /* CPTR_EL3. Check both EZ and TFP. */
+ if (!(env->cp15.cptr_el[3] & CPTR_EZ)
+ || (env->cp15.cptr_el[3] & CPTR_TFP)) {
+ return 3;
+ }
+#endif
+ return 0;
+}
+
+static CPAccessResult zcr_access(CPUARMState *env, const ARMCPRegInfo *ri,
+ bool isread)
+{
+ switch (sve_exception_el(env)) {
+ case 3:
+ return CP_ACCESS_TRAP_EL3;
+ case 2:
+ return CP_ACCESS_TRAP_EL2;
+ case 1:
+ return CP_ACCESS_TRAP;
+ }
+ return CP_ACCESS_OK;
+}
+
+static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri,
+ uint64_t value)
+{
+ /* Bits other than [3:0] are RAZ/WI. */
+ raw_write(env, ri, value & 0xf);
+}
+
+static const ARMCPRegInfo zcr_el1_reginfo = {
+ .name = "ZCR_EL1", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 0, .crn = 1, .crm = 2, .opc2 = 0,
+ .access = PL1_RW, .accessfn = zcr_access, .type = ARM_CP_64BIT,
+ .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[1]),
+ .writefn = zcr_write, .raw_writefn = raw_write
+};
+
+static const ARMCPRegInfo zcr_el2_reginfo = {
+ .name = "ZCR_EL2", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0,
+ .access = PL2_RW, .accessfn = zcr_access, .type = ARM_CP_64BIT,
+ .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[2]),
+ .writefn = zcr_write, .raw_writefn = raw_write
+};
+
+static const ARMCPRegInfo zcr_no_el2_reginfo = {
+ .name = "ZCR_EL2", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 4, .crn = 1, .crm = 2, .opc2 = 0,
+ .access = PL2_RW, .type = ARM_CP_64BIT,
+ .readfn = arm_cp_read_zero, .writefn = arm_cp_write_ignore
+};
+
+static const ARMCPRegInfo zcr_el3_reginfo = {
+ .name = "ZCR_EL3", .state = ARM_CP_STATE_AA64,
+ .opc0 = 3, .opc1 = 6, .crn = 1, .crm = 2, .opc2 = 0,
+ .access = PL3_RW, .accessfn = zcr_access, .type = ARM_CP_64BIT,
+ .fieldoffset = offsetof(CPUARMState, vfp.zcr_el[3]),
+ .writefn = zcr_write, .raw_writefn = raw_write
+};
+
void hw_watchpoint_update(ARMCPU *cpu, int n)
{
CPUARMState *env = &cpu->env;
@@ -5332,6 +5451,18 @@ void register_cp_regs_for_features(ARMCPU *cpu)
}
define_one_arm_cp_reg(cpu, &sctlr);
}
+
+ if (arm_feature(env, ARM_FEATURE_SVE)) {
+ define_one_arm_cp_reg(cpu, &zcr_el1_reginfo);
+ if (arm_feature(env, ARM_FEATURE_EL2)) {
+ define_one_arm_cp_reg(cpu, &zcr_el2_reginfo);
+ } else {
+ define_one_arm_cp_reg(cpu, &zcr_no_el2_reginfo);
+ }
+ if (arm_feature(env, ARM_FEATURE_EL3)) {
+ define_one_arm_cp_reg(cpu, &zcr_el3_reginfo);
+ }
+ }
}
void arm_cpu_register_gdb_regs_for_features(ARMCPU *cpu)
@@ -6161,12 +6292,127 @@ uint32_t arm_phys_excp_target_el(CPUState *cs, uint32_t excp_idx,
return target_el;
}
-static void v7m_push(CPUARMState *env, uint32_t val)
+static bool v7m_stack_write(ARMCPU *cpu, uint32_t addr, uint32_t value,
+ ARMMMUIdx mmu_idx, bool ignfault)
{
- CPUState *cs = CPU(arm_env_get_cpu(env));
+ CPUState *cs = CPU(cpu);
+ CPUARMState *env = &cpu->env;
+ MemTxAttrs attrs = {};
+ MemTxResult txres;
+ target_ulong page_size;
+ hwaddr physaddr;
+ int prot;
+ ARMMMUFaultInfo fi;
+ bool secure = mmu_idx & ARM_MMU_IDX_M_S;
+ int exc;
+ bool exc_secure;
+
+ if (get_phys_addr(env, addr, MMU_DATA_STORE, mmu_idx, &physaddr,
+ &attrs, &prot, &page_size, &fi, NULL)) {
+ /* MPU/SAU lookup failed */
+ if (fi.type == ARMFault_QEMU_SFault) {
+ qemu_log_mask(CPU_LOG_INT,
+ "...SecureFault with SFSR.AUVIOL during stacking\n");
+ env->v7m.sfsr |= R_V7M_SFSR_AUVIOL_MASK | R_V7M_SFSR_SFARVALID_MASK;
+ env->v7m.sfar = addr;
+ exc = ARMV7M_EXCP_SECURE;
+ exc_secure = false;
+ } else {
+ qemu_log_mask(CPU_LOG_INT, "...MemManageFault with CFSR.MSTKERR\n");
+ env->v7m.cfsr[secure] |= R_V7M_CFSR_MSTKERR_MASK;
+ exc = ARMV7M_EXCP_MEM;
+ exc_secure = secure;
+ }
+ goto pend_fault;
+ }
+ address_space_stl_le(arm_addressspace(cs, attrs), physaddr, value,
+ attrs, &txres);
+ if (txres != MEMTX_OK) {
+ /* BusFault trying to write the data */
+ qemu_log_mask(CPU_LOG_INT, "...BusFault with BFSR.STKERR\n");
+ env->v7m.cfsr[M_REG_NS] |= R_V7M_CFSR_STKERR_MASK;
+ exc = ARMV7M_EXCP_BUS;
+ exc_secure = false;
+ goto pend_fault;
+ }
+ return true;
+
+pend_fault:
+ /* By pending the exception at this point we are making
+ * the IMPDEF choice "overridden exceptions pended" (see the
+ * MergeExcInfo() pseudocode). The other choice would be to not
+ * pend them now and then make a choice about which to throw away
+ * later if we have two derived exceptions.
+ * The only case when we must not pend the exception but instead
+ * throw it away is if we are doing the push of the callee registers
+ * and we've already generated a derived exception. Even in this
+ * case we will still update the fault status registers.
+ */
+ if (!ignfault) {
+ armv7m_nvic_set_pending_derived(env->nvic, exc, exc_secure);
+ }
+ return false;
+}
+
+static bool v7m_stack_read(ARMCPU *cpu, uint32_t *dest, uint32_t addr,
+ ARMMMUIdx mmu_idx)
+{
+ CPUState *cs = CPU(cpu);
+ CPUARMState *env = &cpu->env;
+ MemTxAttrs attrs = {};
+ MemTxResult txres;
+ target_ulong page_size;
+ hwaddr physaddr;
+ int prot;
+ ARMMMUFaultInfo fi;
+ bool secure = mmu_idx & ARM_MMU_IDX_M_S;
+ int exc;
+ bool exc_secure;
+ uint32_t value;
+
+ if (get_phys_addr(env, addr, MMU_DATA_LOAD, mmu_idx, &physaddr,
+ &attrs, &prot, &page_size, &fi, NULL)) {
+ /* MPU/SAU lookup failed */
+ if (fi.type == ARMFault_QEMU_SFault) {
+ qemu_log_mask(CPU_LOG_INT,
+ "...SecureFault with SFSR.AUVIOL during unstack\n");
+ env->v7m.sfsr |= R_V7M_SFSR_AUVIOL_MASK | R_V7M_SFSR_SFARVALID_MASK;
+ env->v7m.sfar = addr;
+ exc = ARMV7M_EXCP_SECURE;
+ exc_secure = false;
+ } else {
+ qemu_log_mask(CPU_LOG_INT,
+ "...MemManageFault with CFSR.MUNSTKERR\n");
+ env->v7m.cfsr[secure] |= R_V7M_CFSR_MUNSTKERR_MASK;
+ exc = ARMV7M_EXCP_MEM;
+ exc_secure = secure;
+ }
+ goto pend_fault;
+ }
- env->regs[13] -= 4;
- stl_phys(cs->as, env->regs[13], val);
+ value = address_space_ldl(arm_addressspace(cs, attrs), physaddr,
+ attrs, &txres);
+ if (txres != MEMTX_OK) {
+ /* BusFault trying to read the data */
+ qemu_log_mask(CPU_LOG_INT, "...BusFault with BFSR.UNSTKERR\n");
+ env->v7m.cfsr[M_REG_NS] |= R_V7M_CFSR_UNSTKERR_MASK;
+ exc = ARMV7M_EXCP_BUS;
+ exc_secure = false;
+ goto pend_fault;
+ }
+
+ *dest = value;
+ return true;
+
+pend_fault:
+ /* By pending the exception at this point we are making
+ * the IMPDEF choice "overridden exceptions pended" (see the
+ * MergeExcInfo() pseudocode). The other choice would be to not
+ * pend them now and then make a choice about which to throw away
+ * later if we have two derived exceptions.
+ */
+ armv7m_nvic_set_pending(env->nvic, exc, exc_secure);
+ return false;
}
/* Return true if we're using the process stack pointer (not the MSP) */
@@ -6395,65 +6641,126 @@ static uint32_t *get_v7m_sp_ptr(CPUARMState *env, bool secure, bool threadmode,
}
}
-static uint32_t arm_v7m_load_vector(ARMCPU *cpu, bool targets_secure)
+static bool arm_v7m_load_vector(ARMCPU *cpu, int exc, bool targets_secure,
+ uint32_t *pvec)
{
CPUState *cs = CPU(cpu);
CPUARMState *env = &cpu->env;
MemTxResult result;
- hwaddr vec = env->v7m.vecbase[targets_secure] + env->v7m.exception * 4;
- uint32_t addr;
+ uint32_t addr = env->v7m.vecbase[targets_secure] + exc * 4;
+ uint32_t vector_entry;
+ MemTxAttrs attrs = {};
+ ARMMMUIdx mmu_idx;
+ bool exc_secure;
+
+ mmu_idx = arm_v7m_mmu_idx_for_secstate_and_priv(env, targets_secure, true);
+
+ /* We don't do a get_phys_addr() here because the rules for vector
+ * loads are special: they always use the default memory map, and
+ * the default memory map permits reads from all addresses.
+ * Since there's no easy way to pass through to pmsav8_mpu_lookup()
+ * that we want this special case which would always say "yes",
+ * we just do the SAU lookup here followed by a direct physical load.
+ */
+ attrs.secure = targets_secure;
+ attrs.user = false;
- addr = address_space_ldl(cs->as, vec,
- MEMTXATTRS_UNSPECIFIED, &result);
+ if (arm_feature(env, ARM_FEATURE_M_SECURITY)) {
+ V8M_SAttributes sattrs = {};
+
+ v8m_security_lookup(env, addr, MMU_DATA_LOAD, mmu_idx, &sattrs);
+ if (sattrs.ns) {
+ attrs.secure = false;
+ } else if (!targets_secure) {
+ /* NS access to S memory */
+ goto load_fail;
+ }
+ }
+
+ vector_entry = address_space_ldl(arm_addressspace(cs, attrs), addr,
+ attrs, &result);
if (result != MEMTX_OK) {
- /* Architecturally this should cause a HardFault setting HSFR.VECTTBL,
- * which would then be immediately followed by our failing to load
- * the entry vector for that HardFault, which is a Lockup case.
- * Since we don't model Lockup, we just report this guest error
- * via cpu_abort().
- */
- cpu_abort(cs, "Failed to read from %s exception vector table "
- "entry %08x\n", targets_secure ? "secure" : "nonsecure",
- (unsigned)vec);
+ goto load_fail;
}
- return addr;
+ *pvec = vector_entry;
+ return true;
+
+load_fail:
+ /* All vector table fetch fails are reported as HardFault, with
+ * HFSR.VECTTBL and .FORCED set. (FORCED is set because
+ * technically the underlying exception is a MemManage or BusFault
+ * that is escalated to HardFault.) This is a terminal exception,
+ * so we will either take the HardFault immediately or else enter
+ * lockup (the latter case is handled in armv7m_nvic_set_pending_derived()).
+ */
+ exc_secure = targets_secure ||
+ !(cpu->env.v7m.aircr & R_V7M_AIRCR_BFHFNMINS_MASK);
+ env->v7m.hfsr |= R_V7M_HFSR_VECTTBL_MASK | R_V7M_HFSR_FORCED_MASK;
+ armv7m_nvic_set_pending_derived(env->nvic, ARMV7M_EXCP_HARD, exc_secure);
+ return false;
}
-static void v7m_push_callee_stack(ARMCPU *cpu, uint32_t lr, bool dotailchain)
+static bool v7m_push_callee_stack(ARMCPU *cpu, uint32_t lr, bool dotailchain,
+ bool ignore_faults)
{
/* For v8M, push the callee-saves register part of the stack frame.
* Compare the v8M pseudocode PushCalleeStack().
* In the tailchaining case this may not be the current stack.
*/
CPUARMState *env = &cpu->env;
- CPUState *cs = CPU(cpu);
uint32_t *frame_sp_p;
uint32_t frameptr;
+ ARMMMUIdx mmu_idx;
+ bool stacked_ok;
if (dotailchain) {
- frame_sp_p = get_v7m_sp_ptr(env, true,
- lr & R_V7M_EXCRET_MODE_MASK,
+ bool mode = lr & R_V7M_EXCRET_MODE_MASK;
+ bool priv = !(env->v7m.control[M_REG_S] & R_V7M_CONTROL_NPRIV_MASK) ||
+ !mode;
+
+ mmu_idx = arm_v7m_mmu_idx_for_secstate_and_priv(env, M_REG_S, priv);
+ frame_sp_p = get_v7m_sp_ptr(env, M_REG_S, mode,
lr & R_V7M_EXCRET_SPSEL_MASK);
} else {
+ mmu_idx = core_to_arm_mmu_idx(env, cpu_mmu_index(env, false));
frame_sp_p = &env->regs[13];
}
frameptr = *frame_sp_p - 0x28;
- stl_phys(cs->as, frameptr, 0xfefa125b);
- stl_phys(cs->as, frameptr + 0x8, env->regs[4]);
- stl_phys(cs->as, frameptr + 0xc, env->regs[5]);
- stl_phys(cs->as, frameptr + 0x10, env->regs[6]);
- stl_phys(cs->as, frameptr + 0x14, env->regs[7]);
- stl_phys(cs->as, frameptr + 0x18, env->regs[8]);
- stl_phys(cs->as, frameptr + 0x1c, env->regs[9]);
- stl_phys(cs->as, frameptr + 0x20, env->regs[10]);
- stl_phys(cs->as, frameptr + 0x24, env->regs[11]);
-
+ /* Write as much of the stack frame as we can. A write failure may
+ * cause us to pend a derived exception.
+ */
+ stacked_ok =
+ v7m_stack_write(cpu, frameptr, 0xfefa125b, mmu_idx, ignore_faults) &&
+ v7m_stack_write(cpu, frameptr + 0x8, env->regs[4], mmu_idx,
+ ignore_faults) &&
+ v7m_stack_write(cpu, frameptr + 0xc, env->regs[5], mmu_idx,
+ ignore_faults) &&
+ v7m_stack_write(cpu, frameptr + 0x10, env->regs[6], mmu_idx,
+ ignore_faults) &&
+ v7m_stack_write(cpu, frameptr + 0x14, env->regs[7], mmu_idx,
+ ignore_faults) &&
+ v7m_stack_write(cpu, frameptr + 0x18, env->regs[8], mmu_idx,
+ ignore_faults) &&
+ v7m_stack_write(cpu, frameptr + 0x1c, env->regs[9], mmu_idx,
+ ignore_faults) &&
+ v7m_stack_write(cpu, frameptr + 0x20, env->regs[10], mmu_idx,
+ ignore_faults) &&
+ v7m_stack_write(cpu, frameptr + 0x24, env->regs[11], mmu_idx,
+ ignore_faults);
+
+ /* Update SP regardless of whether any of the stack accesses failed.
+ * When we implement v8M stack limit checking then this attempt to
+ * update SP might also fail and result in a derived exception.
+ */
*frame_sp_p = frameptr;
+
+ return !stacked_ok;
}
-static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain)
+static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain,
+ bool ignore_stackfaults)
{
/* Do the "take the exception" parts of exception entry,
* but not the pushing of state to the stack. This is
@@ -6462,8 +6769,10 @@ static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain)
CPUARMState *env = &cpu->env;
uint32_t addr;
bool targets_secure;
+ int exc;
+ bool push_failed = false;
- targets_secure = armv7m_nvic_acknowledge_irq(env->nvic);
+ armv7m_nvic_get_pending_irq_info(env->nvic, &exc, &targets_secure);
if (arm_feature(env, ARM_FEATURE_V8)) {
if (arm_feature(env, ARM_FEATURE_M_SECURITY) &&
@@ -6489,7 +6798,8 @@ static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain)
*/
if (lr & R_V7M_EXCRET_DCRS_MASK &&
!(dotailchain && (lr & R_V7M_EXCRET_ES_MASK))) {
- v7m_push_callee_stack(cpu, lr, dotailchain);
+ push_failed = v7m_push_callee_stack(cpu, lr, dotailchain,
+ ignore_stackfaults);
}
lr |= R_V7M_EXCRET_DCRS_MASK;
}
@@ -6531,6 +6841,27 @@ static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain)
}
}
+ if (push_failed && !ignore_stackfaults) {
+ /* Derived exception on callee-saves register stacking:
+ * we might now want to take a different exception which
+ * targets a different security state, so try again from the top.
+ */
+ v7m_exception_taken(cpu, lr, true, true);
+ return;
+ }
+
+ if (!arm_v7m_load_vector(cpu, exc, targets_secure, &addr)) {
+ /* Vector load failed: derived exception */
+ v7m_exception_taken(cpu, lr, true, true);
+ return;
+ }
+
+ /* Now we've done everything that might cause a derived exception
+ * we can go ahead and activate whichever exception we're going to
+ * take (which might now be the derived exception).
+ */
+ armv7m_nvic_acknowledge_irq(env->nvic);
+
/* Switch to target security state -- must do this before writing SPSEL */
switch_v7m_security_state(env, targets_secure);
write_v7m_control_spsel(env, 0);
@@ -6538,34 +6869,55 @@ static void v7m_exception_taken(ARMCPU *cpu, uint32_t lr, bool dotailchain)
/* Clear IT bits */
env->condexec_bits = 0;
env->regs[14] = lr;
- addr = arm_v7m_load_vector(cpu, targets_secure);
env->regs[15] = addr & 0xfffffffe;
env->thumb = addr & 1;
}
-static void v7m_push_stack(ARMCPU *cpu)
+static bool v7m_push_stack(ARMCPU *cpu)
{
/* Do the "set up stack frame" part of exception entry,
* similar to pseudocode PushStack().
+ * Return true if we generate a derived exception (and so
+ * should ignore further stack faults trying to process
+ * that derived exception.)
*/
+ bool stacked_ok;
CPUARMState *env = &cpu->env;
uint32_t xpsr = xpsr_read(env);
+ uint32_t frameptr = env->regs[13];
+ ARMMMUIdx mmu_idx = core_to_arm_mmu_idx(env, cpu_mmu_index(env, false));
/* Align stack pointer if the guest wants that */
- if ((env->regs[13] & 4) &&
+ if ((frameptr & 4) &&
(env->v7m.ccr[env->v7m.secure] & R_V7M_CCR_STKALIGN_MASK)) {
- env->regs[13] -= 4;
+ frameptr -= 4;
xpsr |= XPSR_SPREALIGN;
}
- /* Switch to the handler mode. */
- v7m_push(env, xpsr);
- v7m_push(env, env->regs[15]);
- v7m_push(env, env->regs[14]);
- v7m_push(env, env->regs[12]);
- v7m_push(env, env->regs[3]);
- v7m_push(env, env->regs[2]);
- v7m_push(env, env->regs[1]);
- v7m_push(env, env->regs[0]);
+
+ frameptr -= 0x20;
+
+ /* Write as much of the stack frame as we can. If we fail a stack
+ * write this will result in a derived exception being pended
+ * (which may be taken in preference to the one we started with
+ * if it has higher priority).
+ */
+ stacked_ok =
+ v7m_stack_write(cpu, frameptr, env->regs[0], mmu_idx, false) &&
+ v7m_stack_write(cpu, frameptr + 4, env->regs[1], mmu_idx, false) &&
+ v7m_stack_write(cpu, frameptr + 8, env->regs[2], mmu_idx, false) &&
+ v7m_stack_write(cpu, frameptr + 12, env->regs[3], mmu_idx, false) &&
+ v7m_stack_write(cpu, frameptr + 16, env->regs[12], mmu_idx, false) &&
+ v7m_stack_write(cpu, frameptr + 20, env->regs[14], mmu_idx, false) &&
+ v7m_stack_write(cpu, frameptr + 24, env->regs[15], mmu_idx, false) &&
+ v7m_stack_write(cpu, frameptr + 28, xpsr, mmu_idx, false);
+
+ /* Update SP regardless of whether any of the stack accesses failed.
+ * When we implement v8M stack limit checking then this attempt to
+ * update SP might also fail and result in a derived exception.
+ */
+ env->regs[13] = frameptr;
+
+ return !stacked_ok;
}
static void do_v7m_exception_exit(ARMCPU *cpu)
@@ -6711,7 +7063,7 @@ static void do_v7m_exception_exit(ARMCPU *cpu)
if (sfault) {
env->v7m.sfsr |= R_V7M_SFSR_INVER_MASK;
armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SECURE, false);
- v7m_exception_taken(cpu, excret, true);
+ v7m_exception_taken(cpu, excret, true, false);
qemu_log_mask(CPU_LOG_INT, "...taking SecureFault on existing "
"stackframe: failed EXC_RETURN.ES validity check\n");
return;
@@ -6723,7 +7075,7 @@ static void do_v7m_exception_exit(ARMCPU *cpu)
*/
env->v7m.cfsr[env->v7m.secure] |= R_V7M_CFSR_INVPC_MASK;
armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE, env->v7m.secure);
- v7m_exception_taken(cpu, excret, true);
+ v7m_exception_taken(cpu, excret, true, false);
qemu_log_mask(CPU_LOG_INT, "...taking UsageFault on existing "
"stackframe: failed exception return integrity check\n");
return;
@@ -6752,6 +7104,11 @@ static void do_v7m_exception_exit(ARMCPU *cpu)
!return_to_handler,
return_to_sp_process);
uint32_t frameptr = *frame_sp_p;
+ bool pop_ok = true;
+ ARMMMUIdx mmu_idx;
+
+ mmu_idx = arm_v7m_mmu_idx_for_secstate_and_priv(env, return_to_secure,
+ !return_to_handler);
if (!QEMU_IS_ALIGNED(frameptr, 8) &&
arm_feature(env, ARM_FEATURE_V8)) {
@@ -6771,36 +7128,45 @@ static void do_v7m_exception_exit(ARMCPU *cpu)
/* Take a SecureFault on the current stack */
env->v7m.sfsr |= R_V7M_SFSR_INVIS_MASK;
armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_SECURE, false);
- v7m_exception_taken(cpu, excret, true);
+ v7m_exception_taken(cpu, excret, true, false);
qemu_log_mask(CPU_LOG_INT, "...taking SecureFault on existing "
"stackframe: failed exception return integrity "
"signature check\n");
return;
}
- env->regs[4] = ldl_phys(cs->as, frameptr + 0x8);
- env->regs[5] = ldl_phys(cs->as, frameptr + 0xc);
- env->regs[6] = ldl_phys(cs->as, frameptr + 0x10);
- env->regs[7] = ldl_phys(cs->as, frameptr + 0x14);
- env->regs[8] = ldl_phys(cs->as, frameptr + 0x18);
- env->regs[9] = ldl_phys(cs->as, frameptr + 0x1c);
- env->regs[10] = ldl_phys(cs->as, frameptr + 0x20);
- env->regs[11] = ldl_phys(cs->as, frameptr + 0x24);
+ pop_ok =
+ v7m_stack_read(cpu, &env->regs[4], frameptr + 0x8, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[4], frameptr + 0x8, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[5], frameptr + 0xc, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[6], frameptr + 0x10, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[7], frameptr + 0x14, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[8], frameptr + 0x18, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[9], frameptr + 0x1c, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[10], frameptr + 0x20, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[11], frameptr + 0x24, mmu_idx);
frameptr += 0x28;
}
- /* Pop registers. TODO: make these accesses use the correct
- * attributes and address space (S/NS, priv/unpriv) and handle
- * memory transaction failures.
- */
- env->regs[0] = ldl_phys(cs->as, frameptr);
- env->regs[1] = ldl_phys(cs->as, frameptr + 0x4);
- env->regs[2] = ldl_phys(cs->as, frameptr + 0x8);
- env->regs[3] = ldl_phys(cs->as, frameptr + 0xc);
- env->regs[12] = ldl_phys(cs->as, frameptr + 0x10);
- env->regs[14] = ldl_phys(cs->as, frameptr + 0x14);
- env->regs[15] = ldl_phys(cs->as, frameptr + 0x18);
+ /* Pop registers */
+ pop_ok = pop_ok &&
+ v7m_stack_read(cpu, &env->regs[0], frameptr, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[1], frameptr + 0x4, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[2], frameptr + 0x8, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[3], frameptr + 0xc, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[12], frameptr + 0x10, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[14], frameptr + 0x14, mmu_idx) &&
+ v7m_stack_read(cpu, &env->regs[15], frameptr + 0x18, mmu_idx) &&
+ v7m_stack_read(cpu, &xpsr, frameptr + 0x1c, mmu_idx);
+
+ if (!pop_ok) {
+ /* v7m_stack_read() pended a fault, so take it (as a tail
+ * chained exception on the same stack frame)
+ */
+ v7m_exception_taken(cpu, excret, true, false);
+ return;
+ }
/* Returning from an exception with a PC with bit 0 set is defined
* behaviour on v8M (bit 0 is ignored), but for v7M it was specified
@@ -6819,8 +7185,6 @@ static void do_v7m_exception_exit(ARMCPU *cpu)
}
}
- xpsr = ldl_phys(cs->as, frameptr + 0x1c);
-
if (arm_feature(env, ARM_FEATURE_V8)) {
/* For v8M we have to check whether the xPSR exception field
* matches the EXCRET value for return to handler/thread
@@ -6836,7 +7200,7 @@ static void do_v7m_exception_exit(ARMCPU *cpu)
armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE,
env->v7m.secure);
env->v7m.cfsr[env->v7m.secure] |= R_V7M_CFSR_INVPC_MASK;
- v7m_exception_taken(cpu, excret, true);
+ v7m_exception_taken(cpu, excret, true, false);
qemu_log_mask(CPU_LOG_INT, "...taking UsageFault on existing "
"stackframe: failed exception return integrity "
"check\n");
@@ -6869,11 +7233,13 @@ static void do_v7m_exception_exit(ARMCPU *cpu)
/* Take an INVPC UsageFault by pushing the stack again;
* we know we're v7M so this is never a Secure UsageFault.
*/
+ bool ignore_stackfaults;
+
assert(!arm_feature(env, ARM_FEATURE_V8));
armv7m_nvic_set_pending(env->nvic, ARMV7M_EXCP_USAGE, false);
env->v7m.cfsr[env->v7m.secure] |= R_V7M_CFSR_INVPC_MASK;
- v7m_push_stack(cpu);
- v7m_exception_taken(cpu, excret, false);
+ ignore_stackfaults = v7m_push_stack(cpu);
+ v7m_exception_taken(cpu, excret, false, ignore_stackfaults);
qemu_log_mask(CPU_LOG_INT, "...taking UsageFault on new stackframe: "
"failed exception return integrity check\n");
return;
@@ -7114,6 +7480,7 @@ void arm_v7m_cpu_do_interrupt(CPUState *cs)
ARMCPU *cpu = ARM_CPU(cs);
CPUARMState *env = &cpu->env;
uint32_t lr;
+ bool ignore_stackfaults;
arm_log_exception(cs->exception_index);
@@ -7288,8 +7655,8 @@ void arm_v7m_cpu_do_interrupt(CPUState *cs)
lr |= R_V7M_EXCRET_MODE_MASK;
}
- v7m_push_stack(cpu);
- v7m_exception_taken(cpu, lr, false);
+ ignore_stackfaults = v7m_push_stack(cpu);
+ v7m_exception_taken(cpu, lr, false, ignore_stackfaults);
qemu_log_mask(CPU_LOG_INT, "... as %d\n", env->v7m.exception);
}
@@ -11692,14 +12059,37 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
target_ulong *cs_base, uint32_t *pflags)
{
ARMMMUIdx mmu_idx = core_to_arm_mmu_idx(env, cpu_mmu_index(env, false));
+ int fp_el = fp_exception_el(env);
uint32_t flags;
if (is_a64(env)) {
+ int sve_el = sve_exception_el(env);
+ uint32_t zcr_len;
+
*pc = env->pc;
flags = ARM_TBFLAG_AARCH64_STATE_MASK;
/* Get control bits for tagged addresses */
flags |= (arm_regime_tbi0(env, mmu_idx) << ARM_TBFLAG_TBI0_SHIFT);
flags |= (arm_regime_tbi1(env, mmu_idx) << ARM_TBFLAG_TBI1_SHIFT);
+ flags |= sve_el << ARM_TBFLAG_SVEEXC_EL_SHIFT;
+
+ /* If SVE is disabled, but FP is enabled,
+ then the effective len is 0. */
+ if (sve_el != 0 && fp_el == 0) {
+ zcr_len = 0;
+ } else {
+ int current_el = arm_current_el(env);
+
+ zcr_len = env->vfp.zcr_el[current_el <= 1 ? 1 : current_el];
+ zcr_len &= 0xf;
+ if (current_el < 2 && arm_feature(env, ARM_FEATURE_EL2)) {
+ zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[2]);
+ }
+ if (current_el < 3 && arm_feature(env, ARM_FEATURE_EL3)) {
+ zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[3]);
+ }
+ }
+ flags |= zcr_len << ARM_TBFLAG_ZCR_LEN_SHIFT;
} else {
*pc = env->regs[15];
flags = (env->thumb << ARM_TBFLAG_THUMB_SHIFT)
@@ -11742,7 +12132,7 @@ void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
if (arm_cpu_data_is_big_endian(env)) {
flags |= ARM_TBFLAG_BE_DATA_MASK;
}
- flags |= fp_exception_el(env) << ARM_TBFLAG_FPEXC_EL_SHIFT;
+ flags |= fp_el << ARM_TBFLAG_FPEXC_EL_SHIFT;
if (arm_v7m_is_handler_mode(env)) {
flags |= ARM_TBFLAG_HANDLER_MASK;
diff --git a/target/arm/helper.h b/target/arm/helper.h
index 5dec2e6262..6383d7d09e 100644
--- a/target/arm/helper.h
+++ b/target/arm/helper.h
@@ -534,6 +534,18 @@ DEF_HELPER_FLAGS_3(crypto_sha256h2, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
DEF_HELPER_FLAGS_2(crypto_sha256su0, TCG_CALL_NO_RWG, void, ptr, ptr)
DEF_HELPER_FLAGS_3(crypto_sha256su1, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_3(crypto_sha512h, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_3(crypto_sha512h2, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_2(crypto_sha512su0, TCG_CALL_NO_RWG, void, ptr, ptr)
+DEF_HELPER_FLAGS_3(crypto_sha512su1, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
+
+DEF_HELPER_FLAGS_5(crypto_sm3tt, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32, i32)
+DEF_HELPER_FLAGS_3(crypto_sm3partw1, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
+DEF_HELPER_FLAGS_3(crypto_sm3partw2, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
+
+DEF_HELPER_FLAGS_2(crypto_sm4e, TCG_CALL_NO_RWG, void, ptr, ptr)
+DEF_HELPER_FLAGS_3(crypto_sm4ekey, TCG_CALL_NO_RWG, void, ptr, ptr, ptr)
+
DEF_HELPER_FLAGS_3(crc32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
DEF_HELPER_FLAGS_3(crc32c, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
DEF_HELPER_2(dc_zva, void, env, i64)
diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h
index ff53e9fafb..cfb7e5af72 100644
--- a/target/arm/kvm_arm.h
+++ b/target/arm/kvm_arm.h
@@ -234,6 +234,10 @@ static inline const char *gicv3_class_name(void)
exit(1);
#endif
} else {
+ if (kvm_enabled()) {
+ error_report("Userspace GICv3 is not supported with KVM");
+ exit(1);
+ }
return "arm-gicv3";
}
}
diff --git a/target/arm/machine.c b/target/arm/machine.c
index a85c2430d3..2c8b43062f 100644
--- a/target/arm/machine.c
+++ b/target/arm/machine.c
@@ -50,7 +50,40 @@ static const VMStateDescription vmstate_vfp = {
.minimum_version_id = 3,
.needed = vfp_needed,
.fields = (VMStateField[]) {
- VMSTATE_UINT64_ARRAY(env.vfp.regs, ARMCPU, 64),
+ /* For compatibility, store Qn out of Zn here. */
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[0].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[1].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[2].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[3].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[4].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[5].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[6].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[7].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[8].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[9].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[10].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[11].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[12].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[13].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[14].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[15].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[16].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[17].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[18].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[19].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[20].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[21].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[22].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[23].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[24].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[25].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[26].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[27].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[28].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[29].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[30].d, ARMCPU, 0, 2),
+ VMSTATE_UINT64_SUB_ARRAY(env.vfp.zregs[31].d, ARMCPU, 0, 2),
+
/* The xregs array is a little awkward because element 1 (FPSCR)
* requires a specific accessor, so we have to split it up in
* the vmstate:
@@ -89,6 +122,56 @@ static const VMStateDescription vmstate_iwmmxt = {
}
};
+#ifdef TARGET_AARCH64
+/* The expression ARM_MAX_VQ - 2 is 0 for pure AArch32 build,
+ * and ARMPredicateReg is actively empty. This triggers errors
+ * in the expansion of the VMSTATE macros.
+ */
+
+static bool sve_needed(void *opaque)
+{
+ ARMCPU *cpu = opaque;
+ CPUARMState *env = &cpu->env;
+
+ return arm_feature(env, ARM_FEATURE_SVE);
+}
+
+/* The first two words of each Zreg is stored in VFP state. */
+static const VMStateDescription vmstate_zreg_hi_reg = {
+ .name = "cpu/sve/zreg_hi",
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .fields = (VMStateField[]) {
+ VMSTATE_UINT64_SUB_ARRAY(d, ARMVectorReg, 2, ARM_MAX_VQ - 2),
+ VMSTATE_END_OF_LIST()
+ }
+};
+
+static const VMStateDescription vmstate_preg_reg = {
+ .name = "cpu/sve/preg",
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .fields = (VMStateField[]) {
+ VMSTATE_UINT64_ARRAY(p, ARMPredicateReg, 2 * ARM_MAX_VQ / 8),
+ VMSTATE_END_OF_LIST()
+ }
+};
+
+static const VMStateDescription vmstate_sve = {
+ .name = "cpu/sve",
+ .version_id = 1,
+ .minimum_version_id = 1,
+ .needed = sve_needed,
+ .fields = (VMStateField[]) {
+ VMSTATE_STRUCT_ARRAY(env.vfp.zregs, ARMCPU, 32, 0,
+ vmstate_zreg_hi_reg, ARMVectorReg),
+ VMSTATE_STRUCT_ARRAY(env.vfp.pregs, ARMCPU, 17, 0,
+ vmstate_preg_reg, ARMPredicateReg),
+ VMSTATE_END_OF_LIST()
+ }
+};
+#endif /* AARCH64 */
+
static bool m_needed(void *opaque)
{
ARMCPU *cpu = opaque;
@@ -553,6 +636,9 @@ const VMStateDescription vmstate_arm_cpu = {
&vmstate_pmsav7,
&vmstate_pmsav8,
&vmstate_m_security,
+#ifdef TARGET_AARCH64
+ &vmstate_sve,
+#endif
NULL
}
};
diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c
index eed64c73e5..fb1a4cb532 100644
--- a/target/arm/translate-a64.c
+++ b/target/arm/translate-a64.c
@@ -21,6 +21,7 @@
#include "cpu.h"
#include "exec/exec-all.h"
#include "tcg-op.h"
+#include "tcg-op-gvec.h"
#include "qemu/log.h"
#include "arm_ldst.h"
#include "translate.h"
@@ -84,6 +85,13 @@ typedef void CryptoTwoOpFn(TCGv_ptr, TCGv_ptr);
typedef void CryptoThreeOpIntFn(TCGv_ptr, TCGv_ptr, TCGv_i32);
typedef void CryptoThreeOpFn(TCGv_ptr, TCGv_ptr, TCGv_ptr);
+/* Note that the gvec expanders operate on offsets + sizes. */
+typedef void GVecGen2Fn(unsigned, uint32_t, uint32_t, uint32_t, uint32_t);
+typedef void GVecGen2iFn(unsigned, uint32_t, uint32_t, int64_t,
+ uint32_t, uint32_t);
+typedef void GVecGen3Fn(unsigned, uint32_t, uint32_t,
+ uint32_t, uint32_t, uint32_t);
+
/* initialize TCG globals. */
void a64_translate_init(void)
{
@@ -517,8 +525,8 @@ static inline int vec_reg_offset(DisasContext *s, int regno,
{
int offs = 0;
#ifdef HOST_WORDS_BIGENDIAN
- /* This is complicated slightly because vfp.regs[2n] is
- * still the low half and vfp.regs[2n+1] the high half
+ /* This is complicated slightly because vfp.zregs[n].d[0] is
+ * still the low half and vfp.zregs[n].d[1] the high half
* of the 128 bit vector, even on big endian systems.
* Calculate the offset assuming a fully bigendian 128 bits,
* then XOR to account for the order of the two 64 bit halves.
@@ -528,7 +536,7 @@ static inline int vec_reg_offset(DisasContext *s, int regno,
#else
offs += element * (1 << size);
#endif
- offs += offsetof(CPUARMState, vfp.regs[regno * 2]);
+ offs += offsetof(CPUARMState, vfp.zregs[regno]);
assert_fp_access_checked(s);
return offs;
}
@@ -537,7 +545,7 @@ static inline int vec_reg_offset(DisasContext *s, int regno,
static inline int vec_full_reg_offset(DisasContext *s, int regno)
{
assert_fp_access_checked(s);
- return offsetof(CPUARMState, vfp.regs[regno * 2]);
+ return offsetof(CPUARMState, vfp.zregs[regno]);
}
/* Return a newly allocated pointer to the vector register. */
@@ -548,6 +556,14 @@ static TCGv_ptr vec_full_reg_ptr(DisasContext *s, int regno)
return ret;
}
+/* Return the byte size of the "whole" vector register, VL / 8. */
+static inline int vec_full_reg_size(DisasContext *s)
+{
+ /* FIXME SVE: We should put the composite ZCR_EL* value into tb->flags.
+ In the meantime this is just the AdvSIMD length of 128. */
+ return 128 / 8;
+}
+
/* Return the offset into CPUARMState of a slice (from
* the least significant end) of FP register Qn (ie
* Dn, Sn, Hn or Bn).
@@ -618,6 +634,51 @@ static TCGv_ptr get_fpstatus_ptr(void)
return statusptr;
}
+/* Expand a 2-operand AdvSIMD vector operation using an expander function. */
+static void gen_gvec_fn2(DisasContext *s, bool is_q, int rd, int rn,
+ GVecGen2Fn *gvec_fn, int vece)
+{
+ gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
+ is_q ? 16 : 8, vec_full_reg_size(s));
+}
+
+/* Expand a 2-operand + immediate AdvSIMD vector operation using
+ * an expander function.
+ */
+static void gen_gvec_fn2i(DisasContext *s, bool is_q, int rd, int rn,
+ int64_t imm, GVecGen2iFn *gvec_fn, int vece)
+{
+ gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
+ imm, is_q ? 16 : 8, vec_full_reg_size(s));
+}
+
+/* Expand a 3-operand AdvSIMD vector operation using an expander function. */
+static void gen_gvec_fn3(DisasContext *s, bool is_q, int rd, int rn, int rm,
+ GVecGen3Fn *gvec_fn, int vece)
+{
+ gvec_fn(vece, vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm), is_q ? 16 : 8, vec_full_reg_size(s));
+}
+
+/* Expand a 2-operand + immediate AdvSIMD vector operation using
+ * an op descriptor.
+ */
+static void gen_gvec_op2i(DisasContext *s, bool is_q, int rd,
+ int rn, int64_t imm, const GVecGen2i *gvec_op)
+{
+ tcg_gen_gvec_2i(vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
+ is_q ? 16 : 8, vec_full_reg_size(s), imm, gvec_op);
+}
+
+/* Expand a 3-operand AdvSIMD vector operation using an op descriptor. */
+static void gen_gvec_op3(DisasContext *s, bool is_q, int rd,
+ int rn, int rm, const GVecGen3 *gvec_op)
+{
+ tcg_gen_gvec_3(vec_full_reg_offset(s, rd), vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm), is_q ? 16 : 8,
+ vec_full_reg_size(s), gvec_op);
+}
+
/* Set ZF and NF based on a 64 bit result. This is alas fiddlier
* than the 32 bit equivalent.
*/
@@ -4566,14 +4627,17 @@ static void handle_fp_1src_double(DisasContext *s, int opcode, int rd, int rn)
TCGv_i64 tcg_op;
TCGv_i64 tcg_res;
+ switch (opcode) {
+ case 0x0: /* FMOV */
+ gen_gvec_fn2(s, false, rd, rn, tcg_gen_gvec_mov, 0);
+ return;
+ }
+
fpst = get_fpstatus_ptr();
tcg_op = read_fp_dreg(s, rn);
tcg_res = tcg_temp_new_i64();
switch (opcode) {
- case 0x0: /* FMOV */
- tcg_gen_mov_i64(tcg_res, tcg_op);
- break;
case 0x1: /* FABS */
gen_helper_vfp_absd(tcg_res, tcg_op);
break;
@@ -5848,10 +5912,7 @@ static void handle_simd_dupe(DisasContext *s, int is_q, int rd, int rn,
int imm5)
{
int size = ctz32(imm5);
- int esize = 8 << size;
- int elements = (is_q ? 128 : 64) / esize;
- int index, i;
- TCGv_i64 tmp;
+ int index = imm5 >> (size + 1);
if (size > 3 || (size == 3 && !is_q)) {
unallocated_encoding(s);
@@ -5862,20 +5923,9 @@ static void handle_simd_dupe(DisasContext *s, int is_q, int rd, int rn,
return;
}
- index = imm5 >> (size + 1);
-
- tmp = tcg_temp_new_i64();
- read_vec_element(s, tmp, rn, index, size);
-
- for (i = 0; i < elements; i++) {
- write_vec_element(s, tmp, rd, i, size);
- }
-
- if (!is_q) {
- clear_vec_high(s, rd);
- }
-
- tcg_temp_free_i64(tmp);
+ tcg_gen_gvec_dup_mem(size, vec_full_reg_offset(s, rd),
+ vec_reg_offset(s, rn, index, size),
+ is_q ? 16 : 8, vec_full_reg_size(s));
}
/* DUP (element, scalar)
@@ -5924,9 +5974,7 @@ static void handle_simd_dupg(DisasContext *s, int is_q, int rd, int rn,
int imm5)
{
int size = ctz32(imm5);
- int esize = 8 << size;
- int elements = (is_q ? 128 : 64)/esize;
- int i = 0;
+ uint32_t dofs, oprsz, maxsz;
if (size > 3 || ((size == 3) && !is_q)) {
unallocated_encoding(s);
@@ -5937,12 +5985,11 @@ static void handle_simd_dupg(DisasContext *s, int is_q, int rd, int rn,
return;
}
- for (i = 0; i < elements; i++) {
- write_vec_element(s, cpu_reg(s, rn), rd, i, size);
- }
- if (!is_q) {
- clear_vec_high(s, rd);
- }
+ dofs = vec_full_reg_offset(s, rd);
+ oprsz = is_q ? 16 : 8;
+ maxsz = vec_full_reg_size(s);
+
+ tcg_gen_gvec_dup_i64(size, dofs, oprsz, maxsz, cpu_reg(s, rn));
}
/* INS (Element)
@@ -6133,8 +6180,6 @@ static void disas_simd_mod_imm(DisasContext *s, uint32_t insn)
bool is_neg = extract32(insn, 29, 1);
bool is_q = extract32(insn, 30, 1);
uint64_t imm = 0;
- TCGv_i64 tcg_rd, tcg_imm;
- int i;
if (o2 != 0 || ((cmode == 0xf) && is_neg && !is_q)) {
unallocated_encoding(s);
@@ -6215,32 +6260,18 @@ static void disas_simd_mod_imm(DisasContext *s, uint32_t insn)
imm = ~imm;
}
- tcg_imm = tcg_const_i64(imm);
- tcg_rd = new_tmp_a64(s);
-
- for (i = 0; i < 2; i++) {
- int foffs = i ? fp_reg_hi_offset(s, rd) : fp_reg_offset(s, rd, MO_64);
-
- if (i == 1 && !is_q) {
- /* non-quad ops clear high half of vector */
- tcg_gen_movi_i64(tcg_rd, 0);
- } else if ((cmode & 0x9) == 0x1 || (cmode & 0xd) == 0x9) {
- tcg_gen_ld_i64(tcg_rd, cpu_env, foffs);
- if (is_neg) {
- /* AND (BIC) */
- tcg_gen_and_i64(tcg_rd, tcg_rd, tcg_imm);
- } else {
- /* ORR */
- tcg_gen_or_i64(tcg_rd, tcg_rd, tcg_imm);
- }
+ if (!((cmode & 0x9) == 0x1 || (cmode & 0xd) == 0x9)) {
+ /* MOVI or MVNI, with MVNI negation handled above. */
+ tcg_gen_gvec_dup64i(vec_full_reg_offset(s, rd), is_q ? 16 : 8,
+ vec_full_reg_size(s), imm);
+ } else {
+ /* ORR or BIC, with BIC negation to AND handled above. */
+ if (is_neg) {
+ gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_andi, MO_64);
} else {
- /* MOVI */
- tcg_gen_mov_i64(tcg_rd, tcg_imm);
+ gen_gvec_fn2i(s, is_q, rd, rd, imm, tcg_gen_gvec_ori, MO_64);
}
- tcg_gen_st_i64(tcg_rd, cpu_env, foffs);
}
-
- tcg_temp_free_i64(tcg_imm);
}
/* AdvSIMD scalar copy
@@ -6485,32 +6516,6 @@ static void handle_shri_with_rndacc(TCGv_i64 tcg_res, TCGv_i64 tcg_src,
}
}
-/* Common SHL/SLI - Shift left with an optional insert */
-static void handle_shli_with_ins(TCGv_i64 tcg_res, TCGv_i64 tcg_src,
- bool insert, int shift)
-{
- if (insert) { /* SLI */
- tcg_gen_deposit_i64(tcg_res, tcg_res, tcg_src, shift, 64 - shift);
- } else { /* SHL */
- tcg_gen_shli_i64(tcg_res, tcg_src, shift);
- }
-}
-
-/* SRI: shift right with insert */
-static void handle_shri_with_ins(TCGv_i64 tcg_res, TCGv_i64 tcg_src,
- int size, int shift)
-{
- int esize = 8 << size;
-
- /* shift count same as element size is valid but does nothing;
- * special case to avoid potential shift by 64.
- */
- if (shift != esize) {
- tcg_gen_shri_i64(tcg_src, tcg_src, shift);
- tcg_gen_deposit_i64(tcg_res, tcg_res, tcg_src, 0, esize - shift);
- }
-}
-
/* SSHR[RA]/USHR[RA] - Scalar shift right (optional rounding/accumulate) */
static void handle_scalar_simd_shri(DisasContext *s,
bool is_u, int immh, int immb,
@@ -6561,7 +6566,14 @@ static void handle_scalar_simd_shri(DisasContext *s,
tcg_rd = (accumulate || insert) ? read_fp_dreg(s, rd) : tcg_temp_new_i64();
if (insert) {
- handle_shri_with_ins(tcg_rd, tcg_rn, size, shift);
+ /* shift count same as element size is valid but does nothing;
+ * special case to avoid potential shift by 64.
+ */
+ int esize = 8 << size;
+ if (shift != esize) {
+ tcg_gen_shri_i64(tcg_rn, tcg_rn, shift);
+ tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, 0, esize - shift);
+ }
} else {
handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
accumulate, is_u, size, shift);
@@ -6599,7 +6611,11 @@ static void handle_scalar_simd_shli(DisasContext *s, bool insert,
tcg_rn = read_fp_dreg(s, rn);
tcg_rd = insert ? read_fp_dreg(s, rd) : tcg_temp_new_i64();
- handle_shli_with_ins(tcg_rd, tcg_rn, insert, shift);
+ if (insert) {
+ tcg_gen_deposit_i64(tcg_rd, tcg_rd, tcg_rn, shift, 64 - shift);
+ } else {
+ tcg_gen_shli_i64(tcg_rd, tcg_rn, shift);
+ }
write_fp_dreg(s, rd, tcg_rd);
@@ -7175,6 +7191,28 @@ static void disas_simd_scalar_three_reg_diff(DisasContext *s, uint32_t insn)
}
}
+/* CMTST : test is "if (X & Y != 0)". */
+static void gen_cmtst_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
+{
+ tcg_gen_and_i32(d, a, b);
+ tcg_gen_setcondi_i32(TCG_COND_NE, d, d, 0);
+ tcg_gen_neg_i32(d, d);
+}
+
+static void gen_cmtst_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
+{
+ tcg_gen_and_i64(d, a, b);
+ tcg_gen_setcondi_i64(TCG_COND_NE, d, d, 0);
+ tcg_gen_neg_i64(d, d);
+}
+
+static void gen_cmtst_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
+{
+ tcg_gen_and_vec(vece, d, a, b);
+ tcg_gen_dupi_vec(vece, a, 0);
+ tcg_gen_cmp_vec(TCG_COND_NE, vece, d, d, a);
+}
+
static void handle_3same_64(DisasContext *s, int opcode, bool u,
TCGv_i64 tcg_rd, TCGv_i64 tcg_rn, TCGv_i64 tcg_rm)
{
@@ -7218,10 +7256,7 @@ static void handle_3same_64(DisasContext *s, int opcode, bool u,
cond = TCG_COND_EQ;
goto do_cmop;
}
- /* CMTST : test is "if (X & Y != 0)". */
- tcg_gen_and_i64(tcg_rd, tcg_rn, tcg_rm);
- tcg_gen_setcondi_i64(TCG_COND_NE, tcg_rd, tcg_rd, 0);
- tcg_gen_neg_i64(tcg_rd, tcg_rd);
+ gen_cmtst_i64(tcg_rd, tcg_rn, tcg_rm);
break;
case 0x8: /* SSHL, USHL */
if (u) {
@@ -8329,16 +8364,195 @@ static void disas_simd_scalar_two_reg_misc(DisasContext *s, uint32_t insn)
}
}
+static void gen_ssra8_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ tcg_gen_vec_sar8i_i64(a, a, shift);
+ tcg_gen_vec_add8_i64(d, d, a);
+}
+
+static void gen_ssra16_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ tcg_gen_vec_sar16i_i64(a, a, shift);
+ tcg_gen_vec_add16_i64(d, d, a);
+}
+
+static void gen_ssra32_i32(TCGv_i32 d, TCGv_i32 a, int32_t shift)
+{
+ tcg_gen_sari_i32(a, a, shift);
+ tcg_gen_add_i32(d, d, a);
+}
+
+static void gen_ssra64_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ tcg_gen_sari_i64(a, a, shift);
+ tcg_gen_add_i64(d, d, a);
+}
+
+static void gen_ssra_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
+{
+ tcg_gen_sari_vec(vece, a, a, sh);
+ tcg_gen_add_vec(vece, d, d, a);
+}
+
+static void gen_usra8_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ tcg_gen_vec_shr8i_i64(a, a, shift);
+ tcg_gen_vec_add8_i64(d, d, a);
+}
+
+static void gen_usra16_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ tcg_gen_vec_shr16i_i64(a, a, shift);
+ tcg_gen_vec_add16_i64(d, d, a);
+}
+
+static void gen_usra32_i32(TCGv_i32 d, TCGv_i32 a, int32_t shift)
+{
+ tcg_gen_shri_i32(a, a, shift);
+ tcg_gen_add_i32(d, d, a);
+}
+
+static void gen_usra64_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ tcg_gen_shri_i64(a, a, shift);
+ tcg_gen_add_i64(d, d, a);
+}
+
+static void gen_usra_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
+{
+ tcg_gen_shri_vec(vece, a, a, sh);
+ tcg_gen_add_vec(vece, d, d, a);
+}
+
+static void gen_shr8_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ uint64_t mask = dup_const(MO_8, 0xff >> shift);
+ TCGv_i64 t = tcg_temp_new_i64();
+
+ tcg_gen_shri_i64(t, a, shift);
+ tcg_gen_andi_i64(t, t, mask);
+ tcg_gen_andi_i64(d, d, ~mask);
+ tcg_gen_or_i64(d, d, t);
+ tcg_temp_free_i64(t);
+}
+
+static void gen_shr16_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ uint64_t mask = dup_const(MO_16, 0xffff >> shift);
+ TCGv_i64 t = tcg_temp_new_i64();
+
+ tcg_gen_shri_i64(t, a, shift);
+ tcg_gen_andi_i64(t, t, mask);
+ tcg_gen_andi_i64(d, d, ~mask);
+ tcg_gen_or_i64(d, d, t);
+ tcg_temp_free_i64(t);
+}
+
+static void gen_shr32_ins_i32(TCGv_i32 d, TCGv_i32 a, int32_t shift)
+{
+ tcg_gen_shri_i32(a, a, shift);
+ tcg_gen_deposit_i32(d, d, a, 0, 32 - shift);
+}
+
+static void gen_shr64_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ tcg_gen_shri_i64(a, a, shift);
+ tcg_gen_deposit_i64(d, d, a, 0, 64 - shift);
+}
+
+static void gen_shr_ins_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
+{
+ uint64_t mask = (2ull << ((8 << vece) - 1)) - 1;
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ TCGv_vec m = tcg_temp_new_vec_matching(d);
+
+ tcg_gen_dupi_vec(vece, m, mask ^ (mask >> sh));
+ tcg_gen_shri_vec(vece, t, a, sh);
+ tcg_gen_and_vec(vece, d, d, m);
+ tcg_gen_or_vec(vece, d, d, t);
+
+ tcg_temp_free_vec(t);
+ tcg_temp_free_vec(m);
+}
+
/* SSHR[RA]/USHR[RA] - Vector shift right (optional rounding/accumulate) */
static void handle_vec_simd_shri(DisasContext *s, bool is_q, bool is_u,
int immh, int immb, int opcode, int rn, int rd)
{
+ static const GVecGen2i ssra_op[4] = {
+ { .fni8 = gen_ssra8_i64,
+ .fniv = gen_ssra_vec,
+ .load_dest = true,
+ .opc = INDEX_op_sari_vec,
+ .vece = MO_8 },
+ { .fni8 = gen_ssra16_i64,
+ .fniv = gen_ssra_vec,
+ .load_dest = true,
+ .opc = INDEX_op_sari_vec,
+ .vece = MO_16 },
+ { .fni4 = gen_ssra32_i32,
+ .fniv = gen_ssra_vec,
+ .load_dest = true,
+ .opc = INDEX_op_sari_vec,
+ .vece = MO_32 },
+ { .fni8 = gen_ssra64_i64,
+ .fniv = gen_ssra_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .opc = INDEX_op_sari_vec,
+ .vece = MO_64 },
+ };
+ static const GVecGen2i usra_op[4] = {
+ { .fni8 = gen_usra8_i64,
+ .fniv = gen_usra_vec,
+ .load_dest = true,
+ .opc = INDEX_op_shri_vec,
+ .vece = MO_8, },
+ { .fni8 = gen_usra16_i64,
+ .fniv = gen_usra_vec,
+ .load_dest = true,
+ .opc = INDEX_op_shri_vec,
+ .vece = MO_16, },
+ { .fni4 = gen_usra32_i32,
+ .fniv = gen_usra_vec,
+ .load_dest = true,
+ .opc = INDEX_op_shri_vec,
+ .vece = MO_32, },
+ { .fni8 = gen_usra64_i64,
+ .fniv = gen_usra_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .opc = INDEX_op_shri_vec,
+ .vece = MO_64, },
+ };
+ static const GVecGen2i sri_op[4] = {
+ { .fni8 = gen_shr8_ins_i64,
+ .fniv = gen_shr_ins_vec,
+ .load_dest = true,
+ .opc = INDEX_op_shri_vec,
+ .vece = MO_8 },
+ { .fni8 = gen_shr16_ins_i64,
+ .fniv = gen_shr_ins_vec,
+ .load_dest = true,
+ .opc = INDEX_op_shri_vec,
+ .vece = MO_16 },
+ { .fni4 = gen_shr32_ins_i32,
+ .fniv = gen_shr_ins_vec,
+ .load_dest = true,
+ .opc = INDEX_op_shri_vec,
+ .vece = MO_32 },
+ { .fni8 = gen_shr64_ins_i64,
+ .fniv = gen_shr_ins_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .opc = INDEX_op_shri_vec,
+ .vece = MO_64 },
+ };
+
int size = 32 - clz32(immh) - 1;
int immhb = immh << 3 | immb;
int shift = 2 * (8 << size) - immhb;
bool accumulate = false;
- bool round = false;
- bool insert = false;
int dsize = is_q ? 128 : 64;
int esize = 8 << size;
int elements = dsize/esize;
@@ -8346,6 +8560,7 @@ static void handle_vec_simd_shri(DisasContext *s, bool is_q, bool is_u,
TCGv_i64 tcg_rn = new_tmp_a64(s);
TCGv_i64 tcg_rd = new_tmp_a64(s);
TCGv_i64 tcg_round;
+ uint64_t round_const;
int i;
if (extract32(immh, 3, 1) && !is_q) {
@@ -8364,64 +8579,159 @@ static void handle_vec_simd_shri(DisasContext *s, bool is_q, bool is_u,
switch (opcode) {
case 0x02: /* SSRA / USRA (accumulate) */
- accumulate = true;
- break;
+ if (is_u) {
+ /* Shift count same as element size produces zero to add. */
+ if (shift == 8 << size) {
+ goto done;
+ }
+ gen_gvec_op2i(s, is_q, rd, rn, shift, &usra_op[size]);
+ } else {
+ /* Shift count same as element size produces all sign to add. */
+ if (shift == 8 << size) {
+ shift -= 1;
+ }
+ gen_gvec_op2i(s, is_q, rd, rn, shift, &ssra_op[size]);
+ }
+ return;
+ case 0x08: /* SRI */
+ /* Shift count same as element size is valid but does nothing. */
+ if (shift == 8 << size) {
+ goto done;
+ }
+ gen_gvec_op2i(s, is_q, rd, rn, shift, &sri_op[size]);
+ return;
+
+ case 0x00: /* SSHR / USHR */
+ if (is_u) {
+ if (shift == 8 << size) {
+ /* Shift count the same size as element size produces zero. */
+ tcg_gen_gvec_dup8i(vec_full_reg_offset(s, rd),
+ is_q ? 16 : 8, vec_full_reg_size(s), 0);
+ } else {
+ gen_gvec_fn2i(s, is_q, rd, rn, shift, tcg_gen_gvec_shri, size);
+ }
+ } else {
+ /* Shift count the same size as element size produces all sign. */
+ if (shift == 8 << size) {
+ shift -= 1;
+ }
+ gen_gvec_fn2i(s, is_q, rd, rn, shift, tcg_gen_gvec_sari, size);
+ }
+ return;
+
case 0x04: /* SRSHR / URSHR (rounding) */
- round = true;
break;
case 0x06: /* SRSRA / URSRA (accum + rounding) */
- accumulate = round = true;
- break;
- case 0x08: /* SRI */
- insert = true;
+ accumulate = true;
break;
+ default:
+ g_assert_not_reached();
}
- if (round) {
- uint64_t round_const = 1ULL << (shift - 1);
- tcg_round = tcg_const_i64(round_const);
- } else {
- tcg_round = NULL;
- }
+ round_const = 1ULL << (shift - 1);
+ tcg_round = tcg_const_i64(round_const);
for (i = 0; i < elements; i++) {
read_vec_element(s, tcg_rn, rn, i, memop);
- if (accumulate || insert) {
+ if (accumulate) {
read_vec_element(s, tcg_rd, rd, i, memop);
}
- if (insert) {
- handle_shri_with_ins(tcg_rd, tcg_rn, size, shift);
- } else {
- handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
- accumulate, is_u, size, shift);
- }
+ handle_shri_with_rndacc(tcg_rd, tcg_rn, tcg_round,
+ accumulate, is_u, size, shift);
write_vec_element(s, tcg_rd, rd, i, size);
}
+ tcg_temp_free_i64(tcg_round);
+ done:
if (!is_q) {
clear_vec_high(s, rd);
}
+}
- if (round) {
- tcg_temp_free_i64(tcg_round);
- }
+static void gen_shl8_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ uint64_t mask = dup_const(MO_8, 0xff << shift);
+ TCGv_i64 t = tcg_temp_new_i64();
+
+ tcg_gen_shli_i64(t, a, shift);
+ tcg_gen_andi_i64(t, t, mask);
+ tcg_gen_andi_i64(d, d, ~mask);
+ tcg_gen_or_i64(d, d, t);
+ tcg_temp_free_i64(t);
+}
+
+static void gen_shl16_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ uint64_t mask = dup_const(MO_16, 0xffff << shift);
+ TCGv_i64 t = tcg_temp_new_i64();
+
+ tcg_gen_shli_i64(t, a, shift);
+ tcg_gen_andi_i64(t, t, mask);
+ tcg_gen_andi_i64(d, d, ~mask);
+ tcg_gen_or_i64(d, d, t);
+ tcg_temp_free_i64(t);
+}
+
+static void gen_shl32_ins_i32(TCGv_i32 d, TCGv_i32 a, int32_t shift)
+{
+ tcg_gen_deposit_i32(d, d, a, shift, 32 - shift);
+}
+
+static void gen_shl64_ins_i64(TCGv_i64 d, TCGv_i64 a, int64_t shift)
+{
+ tcg_gen_deposit_i64(d, d, a, shift, 64 - shift);
+}
+
+static void gen_shl_ins_vec(unsigned vece, TCGv_vec d, TCGv_vec a, int64_t sh)
+{
+ uint64_t mask = (1ull << sh) - 1;
+ TCGv_vec t = tcg_temp_new_vec_matching(d);
+ TCGv_vec m = tcg_temp_new_vec_matching(d);
+
+ tcg_gen_dupi_vec(vece, m, mask);
+ tcg_gen_shli_vec(vece, t, a, sh);
+ tcg_gen_and_vec(vece, d, d, m);
+ tcg_gen_or_vec(vece, d, d, t);
+
+ tcg_temp_free_vec(t);
+ tcg_temp_free_vec(m);
}
/* SHL/SLI - Vector shift left */
static void handle_vec_simd_shli(DisasContext *s, bool is_q, bool insert,
- int immh, int immb, int opcode, int rn, int rd)
+ int immh, int immb, int opcode, int rn, int rd)
{
+ static const GVecGen2i shi_op[4] = {
+ { .fni8 = gen_shl8_ins_i64,
+ .fniv = gen_shl_ins_vec,
+ .opc = INDEX_op_shli_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .vece = MO_8 },
+ { .fni8 = gen_shl16_ins_i64,
+ .fniv = gen_shl_ins_vec,
+ .opc = INDEX_op_shli_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .vece = MO_16 },
+ { .fni4 = gen_shl32_ins_i32,
+ .fniv = gen_shl_ins_vec,
+ .opc = INDEX_op_shli_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .vece = MO_32 },
+ { .fni8 = gen_shl64_ins_i64,
+ .fniv = gen_shl_ins_vec,
+ .opc = INDEX_op_shli_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .vece = MO_64 },
+ };
int size = 32 - clz32(immh) - 1;
int immhb = immh << 3 | immb;
int shift = immhb - (8 << size);
- int dsize = is_q ? 128 : 64;
- int esize = 8 << size;
- int elements = dsize/esize;
- TCGv_i64 tcg_rn = new_tmp_a64(s);
- TCGv_i64 tcg_rd = new_tmp_a64(s);
- int i;
if (extract32(immh, 3, 1) && !is_q) {
unallocated_encoding(s);
@@ -8437,19 +8747,10 @@ static void handle_vec_simd_shli(DisasContext *s, bool is_q, bool insert,
return;
}
- for (i = 0; i < elements; i++) {
- read_vec_element(s, tcg_rn, rn, i, size);
- if (insert) {
- read_vec_element(s, tcg_rd, rd, i, size);
- }
-
- handle_shli_with_ins(tcg_rd, tcg_rn, insert, shift);
-
- write_vec_element(s, tcg_rd, rd, i, size);
- }
-
- if (!is_q) {
- clear_vec_high(s, rd);
+ if (insert) {
+ gen_gvec_op2i(s, is_q, rd, rn, shift, &shi_op[size]);
+ } else {
+ gen_gvec_fn2i(s, is_q, rd, rn, shift, tcg_gen_gvec_shli, size);
}
}
@@ -9072,85 +9373,115 @@ static void disas_simd_three_reg_diff(DisasContext *s, uint32_t insn)
}
}
+static void gen_bsl_i64(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm)
+{
+ tcg_gen_xor_i64(rn, rn, rm);
+ tcg_gen_and_i64(rn, rn, rd);
+ tcg_gen_xor_i64(rd, rm, rn);
+}
+
+static void gen_bit_i64(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm)
+{
+ tcg_gen_xor_i64(rn, rn, rd);
+ tcg_gen_and_i64(rn, rn, rm);
+ tcg_gen_xor_i64(rd, rd, rn);
+}
+
+static void gen_bif_i64(TCGv_i64 rd, TCGv_i64 rn, TCGv_i64 rm)
+{
+ tcg_gen_xor_i64(rn, rn, rd);
+ tcg_gen_andc_i64(rn, rn, rm);
+ tcg_gen_xor_i64(rd, rd, rn);
+}
+
+static void gen_bsl_vec(unsigned vece, TCGv_vec rd, TCGv_vec rn, TCGv_vec rm)
+{
+ tcg_gen_xor_vec(vece, rn, rn, rm);
+ tcg_gen_and_vec(vece, rn, rn, rd);
+ tcg_gen_xor_vec(vece, rd, rm, rn);
+}
+
+static void gen_bit_vec(unsigned vece, TCGv_vec rd, TCGv_vec rn, TCGv_vec rm)
+{
+ tcg_gen_xor_vec(vece, rn, rn, rd);
+ tcg_gen_and_vec(vece, rn, rn, rm);
+ tcg_gen_xor_vec(vece, rd, rd, rn);
+}
+
+static void gen_bif_vec(unsigned vece, TCGv_vec rd, TCGv_vec rn, TCGv_vec rm)
+{
+ tcg_gen_xor_vec(vece, rn, rn, rd);
+ tcg_gen_andc_vec(vece, rn, rn, rm);
+ tcg_gen_xor_vec(vece, rd, rd, rn);
+}
+
/* Logic op (opcode == 3) subgroup of C3.6.16. */
static void disas_simd_3same_logic(DisasContext *s, uint32_t insn)
{
+ static const GVecGen3 bsl_op = {
+ .fni8 = gen_bsl_i64,
+ .fniv = gen_bsl_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true
+ };
+ static const GVecGen3 bit_op = {
+ .fni8 = gen_bit_i64,
+ .fniv = gen_bit_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true
+ };
+ static const GVecGen3 bif_op = {
+ .fni8 = gen_bif_i64,
+ .fniv = gen_bif_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true
+ };
+
int rd = extract32(insn, 0, 5);
int rn = extract32(insn, 5, 5);
int rm = extract32(insn, 16, 5);
int size = extract32(insn, 22, 2);
bool is_u = extract32(insn, 29, 1);
bool is_q = extract32(insn, 30, 1);
- TCGv_i64 tcg_op1, tcg_op2, tcg_res[2];
- int pass;
if (!fp_access_check(s)) {
return;
}
- tcg_op1 = tcg_temp_new_i64();
- tcg_op2 = tcg_temp_new_i64();
- tcg_res[0] = tcg_temp_new_i64();
- tcg_res[1] = tcg_temp_new_i64();
-
- for (pass = 0; pass < (is_q ? 2 : 1); pass++) {
- read_vec_element(s, tcg_op1, rn, pass, MO_64);
- read_vec_element(s, tcg_op2, rm, pass, MO_64);
-
- if (!is_u) {
- switch (size) {
- case 0: /* AND */
- tcg_gen_and_i64(tcg_res[pass], tcg_op1, tcg_op2);
- break;
- case 1: /* BIC */
- tcg_gen_andc_i64(tcg_res[pass], tcg_op1, tcg_op2);
- break;
- case 2: /* ORR */
- tcg_gen_or_i64(tcg_res[pass], tcg_op1, tcg_op2);
- break;
- case 3: /* ORN */
- tcg_gen_orc_i64(tcg_res[pass], tcg_op1, tcg_op2);
- break;
- }
+ switch (size + 4 * is_u) {
+ case 0: /* AND */
+ gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_and, 0);
+ return;
+ case 1: /* BIC */
+ gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_andc, 0);
+ return;
+ case 2: /* ORR */
+ if (rn == rm) { /* MOV */
+ gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_mov, 0);
} else {
- if (size != 0) {
- /* B* ops need res loaded to operate on */
- read_vec_element(s, tcg_res[pass], rd, pass, MO_64);
- }
-
- switch (size) {
- case 0: /* EOR */
- tcg_gen_xor_i64(tcg_res[pass], tcg_op1, tcg_op2);
- break;
- case 1: /* BSL bitwise select */
- tcg_gen_xor_i64(tcg_op1, tcg_op1, tcg_op2);
- tcg_gen_and_i64(tcg_op1, tcg_op1, tcg_res[pass]);
- tcg_gen_xor_i64(tcg_res[pass], tcg_op2, tcg_op1);
- break;
- case 2: /* BIT, bitwise insert if true */
- tcg_gen_xor_i64(tcg_op1, tcg_op1, tcg_res[pass]);
- tcg_gen_and_i64(tcg_op1, tcg_op1, tcg_op2);
- tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
- break;
- case 3: /* BIF, bitwise insert if false */
- tcg_gen_xor_i64(tcg_op1, tcg_op1, tcg_res[pass]);
- tcg_gen_andc_i64(tcg_op1, tcg_op1, tcg_op2);
- tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
- break;
- }
+ gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_or, 0);
}
- }
+ return;
+ case 3: /* ORN */
+ gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_orc, 0);
+ return;
+ case 4: /* EOR */
+ gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_xor, 0);
+ return;
- write_vec_element(s, tcg_res[0], rd, 0, MO_64);
- if (!is_q) {
- tcg_gen_movi_i64(tcg_res[1], 0);
- }
- write_vec_element(s, tcg_res[1], rd, 1, MO_64);
+ case 5: /* BSL bitwise select */
+ gen_gvec_op3(s, is_q, rd, rn, rm, &bsl_op);
+ return;
+ case 6: /* BIT, bitwise insert if true */
+ gen_gvec_op3(s, is_q, rd, rn, rm, &bit_op);
+ return;
+ case 7: /* BIF, bitwise insert if false */
+ gen_gvec_op3(s, is_q, rd, rn, rm, &bif_op);
+ return;
- tcg_temp_free_i64(tcg_op1);
- tcg_temp_free_i64(tcg_op2);
- tcg_temp_free_i64(tcg_res[0]);
- tcg_temp_free_i64(tcg_res[1]);
+ default:
+ g_assert_not_reached();
+ }
}
/* Helper functions for 32 bit comparisons */
@@ -9400,9 +9731,131 @@ static void disas_simd_3same_float(DisasContext *s, uint32_t insn)
}
}
+static void gen_mla8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
+{
+ gen_helper_neon_mul_u8(a, a, b);
+ gen_helper_neon_add_u8(d, d, a);
+}
+
+static void gen_mla16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
+{
+ gen_helper_neon_mul_u16(a, a, b);
+ gen_helper_neon_add_u16(d, d, a);
+}
+
+static void gen_mla32_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
+{
+ tcg_gen_mul_i32(a, a, b);
+ tcg_gen_add_i32(d, d, a);
+}
+
+static void gen_mla64_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
+{
+ tcg_gen_mul_i64(a, a, b);
+ tcg_gen_add_i64(d, d, a);
+}
+
+static void gen_mla_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
+{
+ tcg_gen_mul_vec(vece, a, a, b);
+ tcg_gen_add_vec(vece, d, d, a);
+}
+
+static void gen_mls8_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
+{
+ gen_helper_neon_mul_u8(a, a, b);
+ gen_helper_neon_sub_u8(d, d, a);
+}
+
+static void gen_mls16_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
+{
+ gen_helper_neon_mul_u16(a, a, b);
+ gen_helper_neon_sub_u16(d, d, a);
+}
+
+static void gen_mls32_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
+{
+ tcg_gen_mul_i32(a, a, b);
+ tcg_gen_sub_i32(d, d, a);
+}
+
+static void gen_mls64_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
+{
+ tcg_gen_mul_i64(a, a, b);
+ tcg_gen_sub_i64(d, d, a);
+}
+
+static void gen_mls_vec(unsigned vece, TCGv_vec d, TCGv_vec a, TCGv_vec b)
+{
+ tcg_gen_mul_vec(vece, a, a, b);
+ tcg_gen_sub_vec(vece, d, d, a);
+}
+
/* Integer op subgroup of C3.6.16. */
static void disas_simd_3same_int(DisasContext *s, uint32_t insn)
{
+ static const GVecGen3 cmtst_op[4] = {
+ { .fni4 = gen_helper_neon_tst_u8,
+ .fniv = gen_cmtst_vec,
+ .vece = MO_8 },
+ { .fni4 = gen_helper_neon_tst_u16,
+ .fniv = gen_cmtst_vec,
+ .vece = MO_16 },
+ { .fni4 = gen_cmtst_i32,
+ .fniv = gen_cmtst_vec,
+ .vece = MO_32 },
+ { .fni8 = gen_cmtst_i64,
+ .fniv = gen_cmtst_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .vece = MO_64 },
+ };
+ static const GVecGen3 mla_op[4] = {
+ { .fni4 = gen_mla8_i32,
+ .fniv = gen_mla_vec,
+ .opc = INDEX_op_mul_vec,
+ .load_dest = true,
+ .vece = MO_8 },
+ { .fni4 = gen_mla16_i32,
+ .fniv = gen_mla_vec,
+ .opc = INDEX_op_mul_vec,
+ .load_dest = true,
+ .vece = MO_16 },
+ { .fni4 = gen_mla32_i32,
+ .fniv = gen_mla_vec,
+ .opc = INDEX_op_mul_vec,
+ .load_dest = true,
+ .vece = MO_32 },
+ { .fni8 = gen_mla64_i64,
+ .fniv = gen_mla_vec,
+ .opc = INDEX_op_mul_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .vece = MO_64 },
+ };
+ static const GVecGen3 mls_op[4] = {
+ { .fni4 = gen_mls8_i32,
+ .fniv = gen_mls_vec,
+ .opc = INDEX_op_mul_vec,
+ .load_dest = true,
+ .vece = MO_8 },
+ { .fni4 = gen_mls16_i32,
+ .fniv = gen_mls_vec,
+ .opc = INDEX_op_mul_vec,
+ .load_dest = true,
+ .vece = MO_16 },
+ { .fni4 = gen_mls32_i32,
+ .fniv = gen_mls_vec,
+ .opc = INDEX_op_mul_vec,
+ .load_dest = true,
+ .vece = MO_32 },
+ { .fni8 = gen_mls64_i64,
+ .fniv = gen_mls_vec,
+ .opc = INDEX_op_mul_vec,
+ .prefer_i64 = TCG_TARGET_REG_BITS == 64,
+ .load_dest = true,
+ .vece = MO_64 },
+ };
+
int is_q = extract32(insn, 30, 1);
int u = extract32(insn, 29, 1);
int size = extract32(insn, 22, 2);
@@ -9411,6 +9864,7 @@ static void disas_simd_3same_int(DisasContext *s, uint32_t insn)
int rn = extract32(insn, 5, 5);
int rd = extract32(insn, 0, 5);
int pass;
+ TCGCond cond;
switch (opcode) {
case 0x13: /* MUL, PMUL */
@@ -9450,6 +9904,48 @@ static void disas_simd_3same_int(DisasContext *s, uint32_t insn)
return;
}
+ switch (opcode) {
+ case 0x10: /* ADD, SUB */
+ if (u) {
+ gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_sub, size);
+ } else {
+ gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_add, size);
+ }
+ return;
+ case 0x13: /* MUL, PMUL */
+ if (!u) { /* MUL */
+ gen_gvec_fn3(s, is_q, rd, rn, rm, tcg_gen_gvec_mul, size);
+ return;
+ }
+ break;
+ case 0x12: /* MLA, MLS */
+ if (u) {
+ gen_gvec_op3(s, is_q, rd, rn, rm, &mls_op[size]);
+ } else {
+ gen_gvec_op3(s, is_q, rd, rn, rm, &mla_op[size]);
+ }
+ return;
+ case 0x11:
+ if (!u) { /* CMTST */
+ gen_gvec_op3(s, is_q, rd, rn, rm, &cmtst_op[size]);
+ return;
+ }
+ /* else CMEQ */
+ cond = TCG_COND_EQ;
+ goto do_gvec_cmp;
+ case 0x06: /* CMGT, CMHI */
+ cond = u ? TCG_COND_GTU : TCG_COND_GT;
+ goto do_gvec_cmp;
+ case 0x07: /* CMGE, CMHS */
+ cond = u ? TCG_COND_GEU : TCG_COND_GE;
+ do_gvec_cmp:
+ tcg_gen_gvec_cmp(cond, size, vec_full_reg_offset(s, rd),
+ vec_full_reg_offset(s, rn),
+ vec_full_reg_offset(s, rm),
+ is_q ? 16 : 8, vec_full_reg_size(s));
+ return;
+ }
+
if (size == 3) {
assert(is_q);
for (pass = 0; pass < 2; pass++) {
@@ -9530,26 +10026,6 @@ static void disas_simd_3same_int(DisasContext *s, uint32_t insn)
genenvfn = fns[size][u];
break;
}
- case 0x6: /* CMGT, CMHI */
- {
- static NeonGenTwoOpFn * const fns[3][2] = {
- { gen_helper_neon_cgt_s8, gen_helper_neon_cgt_u8 },
- { gen_helper_neon_cgt_s16, gen_helper_neon_cgt_u16 },
- { gen_helper_neon_cgt_s32, gen_helper_neon_cgt_u32 },
- };
- genfn = fns[size][u];
- break;
- }
- case 0x7: /* CMGE, CMHS */
- {
- static NeonGenTwoOpFn * const fns[3][2] = {
- { gen_helper_neon_cge_s8, gen_helper_neon_cge_u8 },
- { gen_helper_neon_cge_s16, gen_helper_neon_cge_u16 },
- { gen_helper_neon_cge_s32, gen_helper_neon_cge_u32 },
- };
- genfn = fns[size][u];
- break;
- }
case 0x8: /* SSHL, USHL */
{
static NeonGenTwoOpFn * const fns[3][2] = {
@@ -9622,44 +10098,11 @@ static void disas_simd_3same_int(DisasContext *s, uint32_t insn)
genfn = fns[size][u];
break;
}
- case 0x10: /* ADD, SUB */
- {
- static NeonGenTwoOpFn * const fns[3][2] = {
- { gen_helper_neon_add_u8, gen_helper_neon_sub_u8 },
- { gen_helper_neon_add_u16, gen_helper_neon_sub_u16 },
- { tcg_gen_add_i32, tcg_gen_sub_i32 },
- };
- genfn = fns[size][u];
- break;
- }
- case 0x11: /* CMTST, CMEQ */
- {
- static NeonGenTwoOpFn * const fns[3][2] = {
- { gen_helper_neon_tst_u8, gen_helper_neon_ceq_u8 },
- { gen_helper_neon_tst_u16, gen_helper_neon_ceq_u16 },
- { gen_helper_neon_tst_u32, gen_helper_neon_ceq_u32 },
- };
- genfn = fns[size][u];
- break;
- }
case 0x13: /* MUL, PMUL */
- if (u) {
- /* PMUL */
- assert(size == 0);
- genfn = gen_helper_neon_mul_p8;
- break;
- }
- /* fall through : MUL */
- case 0x12: /* MLA, MLS */
- {
- static NeonGenTwoOpFn * const fns[3] = {
- gen_helper_neon_mul_u8,
- gen_helper_neon_mul_u16,
- tcg_gen_mul_i32,
- };
- genfn = fns[size];
+ assert(u); /* PMUL */
+ assert(size == 0);
+ genfn = gen_helper_neon_mul_p8;
break;
- }
case 0x16: /* SQDMULH, SQRDMULH */
{
static NeonGenTwoOpEnvFn * const fns[2][2] = {
@@ -9680,18 +10123,16 @@ static void disas_simd_3same_int(DisasContext *s, uint32_t insn)
genfn(tcg_res, tcg_op1, tcg_op2);
}
- if (opcode == 0xf || opcode == 0x12) {
- /* SABA, UABA, MLA, MLS: accumulating ops */
- static NeonGenTwoOpFn * const fns[3][2] = {
- { gen_helper_neon_add_u8, gen_helper_neon_sub_u8 },
- { gen_helper_neon_add_u16, gen_helper_neon_sub_u16 },
- { tcg_gen_add_i32, tcg_gen_sub_i32 },
+ if (opcode == 0xf) {
+ /* SABA, UABA: accumulating ops */
+ static NeonGenTwoOpFn * const fns[3] = {
+ gen_helper_neon_add_u8,
+ gen_helper_neon_add_u16,
+ tcg_gen_add_i32,
};
- bool is_sub = (opcode == 0x12 && u); /* MLS */
- genfn = fns[size][is_sub];
read_vec_element_i32(s, tcg_op1, rd, pass, MO_32);
- genfn(tcg_res, tcg_op1, tcg_res);
+ fns[size](tcg_res, tcg_op1, tcg_res);
}
write_vec_element_i32(s, tcg_res, rd, pass, MO_32);
@@ -10003,8 +10444,7 @@ static void disas_simd_two_reg_misc(DisasContext *s, uint32_t insn)
return;
case 0x5: /* CNT, NOT, RBIT */
if (u && size == 0) {
- /* NOT: adjust size so we can use the 64-bits-at-a-time loop. */
- size = 3;
+ /* NOT */
break;
} else if (u && size == 1) {
/* RBIT */
@@ -10256,6 +10696,21 @@ static void disas_simd_two_reg_misc(DisasContext *s, uint32_t insn)
tcg_rmode = NULL;
}
+ switch (opcode) {
+ case 0x5:
+ if (u && size == 0) { /* NOT */
+ gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_not, 0);
+ return;
+ }
+ break;
+ case 0xb:
+ if (u) { /* NEG */
+ gen_gvec_fn2(s, is_q, rd, rn, tcg_gen_gvec_neg, size);
+ return;
+ }
+ break;
+ }
+
if (size == 3) {
/* All 64-bit element operations can be shared with scalar 2misc */
int pass;
@@ -11132,6 +11587,341 @@ static void disas_crypto_two_reg_sha(DisasContext *s, uint32_t insn)
tcg_temp_free_ptr(tcg_rn_ptr);
}
+/* Crypto three-reg SHA512
+ * 31 21 20 16 15 14 13 12 11 10 9 5 4 0
+ * +-----------------------+------+---+---+-----+--------+------+------+
+ * | 1 1 0 0 1 1 1 0 0 1 1 | Rm | 1 | O | 0 0 | opcode | Rn | Rd |
+ * +-----------------------+------+---+---+-----+--------+------+------+
+ */
+static void disas_crypto_three_reg_sha512(DisasContext *s, uint32_t insn)
+{
+ int opcode = extract32(insn, 10, 2);
+ int o = extract32(insn, 14, 1);
+ int rm = extract32(insn, 16, 5);
+ int rn = extract32(insn, 5, 5);
+ int rd = extract32(insn, 0, 5);
+ int feature;
+ CryptoThreeOpFn *genfn;
+
+ if (o == 0) {
+ switch (opcode) {
+ case 0: /* SHA512H */
+ feature = ARM_FEATURE_V8_SHA512;
+ genfn = gen_helper_crypto_sha512h;
+ break;
+ case 1: /* SHA512H2 */
+ feature = ARM_FEATURE_V8_SHA512;
+ genfn = gen_helper_crypto_sha512h2;
+ break;
+ case 2: /* SHA512SU1 */
+ feature = ARM_FEATURE_V8_SHA512;
+ genfn = gen_helper_crypto_sha512su1;
+ break;
+ case 3: /* RAX1 */
+ feature = ARM_FEATURE_V8_SHA3;
+ genfn = NULL;
+ break;
+ }
+ } else {
+ switch (opcode) {
+ case 0: /* SM3PARTW1 */
+ feature = ARM_FEATURE_V8_SM3;
+ genfn = gen_helper_crypto_sm3partw1;
+ break;
+ case 1: /* SM3PARTW2 */
+ feature = ARM_FEATURE_V8_SM3;
+ genfn = gen_helper_crypto_sm3partw2;
+ break;
+ case 2: /* SM4EKEY */
+ feature = ARM_FEATURE_V8_SM4;
+ genfn = gen_helper_crypto_sm4ekey;
+ break;
+ default:
+ unallocated_encoding(s);
+ return;
+ }
+ }
+
+ if (!arm_dc_feature(s, feature)) {
+ unallocated_encoding(s);
+ return;
+ }
+
+ if (!fp_access_check(s)) {
+ return;
+ }
+
+ if (genfn) {
+ TCGv_ptr tcg_rd_ptr, tcg_rn_ptr, tcg_rm_ptr;
+
+ tcg_rd_ptr = vec_full_reg_ptr(s, rd);
+ tcg_rn_ptr = vec_full_reg_ptr(s, rn);
+ tcg_rm_ptr = vec_full_reg_ptr(s, rm);
+
+ genfn(tcg_rd_ptr, tcg_rn_ptr, tcg_rm_ptr);
+
+ tcg_temp_free_ptr(tcg_rd_ptr);
+ tcg_temp_free_ptr(tcg_rn_ptr);
+ tcg_temp_free_ptr(tcg_rm_ptr);
+ } else {
+ TCGv_i64 tcg_op1, tcg_op2, tcg_res[2];
+ int pass;
+
+ tcg_op1 = tcg_temp_new_i64();
+ tcg_op2 = tcg_temp_new_i64();
+ tcg_res[0] = tcg_temp_new_i64();
+ tcg_res[1] = tcg_temp_new_i64();
+
+ for (pass = 0; pass < 2; pass++) {
+ read_vec_element(s, tcg_op1, rn, pass, MO_64);
+ read_vec_element(s, tcg_op2, rm, pass, MO_64);
+
+ tcg_gen_rotli_i64(tcg_res[pass], tcg_op2, 1);
+ tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
+ }
+ write_vec_element(s, tcg_res[0], rd, 0, MO_64);
+ write_vec_element(s, tcg_res[1], rd, 1, MO_64);
+
+ tcg_temp_free_i64(tcg_op1);
+ tcg_temp_free_i64(tcg_op2);
+ tcg_temp_free_i64(tcg_res[0]);
+ tcg_temp_free_i64(tcg_res[1]);
+ }
+}
+
+/* Crypto two-reg SHA512
+ * 31 12 11 10 9 5 4 0
+ * +-----------------------------------------+--------+------+------+
+ * | 1 1 0 0 1 1 1 0 1 1 0 0 0 0 0 0 1 0 0 0 | opcode | Rn | Rd |
+ * +-----------------------------------------+--------+------+------+
+ */
+static void disas_crypto_two_reg_sha512(DisasContext *s, uint32_t insn)
+{
+ int opcode = extract32(insn, 10, 2);
+ int rn = extract32(insn, 5, 5);
+ int rd = extract32(insn, 0, 5);
+ TCGv_ptr tcg_rd_ptr, tcg_rn_ptr;
+ int feature;
+ CryptoTwoOpFn *genfn;
+
+ switch (opcode) {
+ case 0: /* SHA512SU0 */
+ feature = ARM_FEATURE_V8_SHA512;
+ genfn = gen_helper_crypto_sha512su0;
+ break;
+ case 1: /* SM4E */
+ feature = ARM_FEATURE_V8_SM4;
+ genfn = gen_helper_crypto_sm4e;
+ break;
+ default:
+ unallocated_encoding(s);
+ return;
+ }
+
+ if (!arm_dc_feature(s, feature)) {
+ unallocated_encoding(s);
+ return;
+ }
+
+ if (!fp_access_check(s)) {
+ return;
+ }
+
+ tcg_rd_ptr = vec_full_reg_ptr(s, rd);
+ tcg_rn_ptr = vec_full_reg_ptr(s, rn);
+
+ genfn(tcg_rd_ptr, tcg_rn_ptr);
+
+ tcg_temp_free_ptr(tcg_rd_ptr);
+ tcg_temp_free_ptr(tcg_rn_ptr);
+}
+
+/* Crypto four-register
+ * 31 23 22 21 20 16 15 14 10 9 5 4 0
+ * +-------------------+-----+------+---+------+------+------+
+ * | 1 1 0 0 1 1 1 0 0 | Op0 | Rm | 0 | Ra | Rn | Rd |
+ * +-------------------+-----+------+---+------+------+------+
+ */
+static void disas_crypto_four_reg(DisasContext *s, uint32_t insn)
+{
+ int op0 = extract32(insn, 21, 2);
+ int rm = extract32(insn, 16, 5);
+ int ra = extract32(insn, 10, 5);
+ int rn = extract32(insn, 5, 5);
+ int rd = extract32(insn, 0, 5);
+ int feature;
+
+ switch (op0) {
+ case 0: /* EOR3 */
+ case 1: /* BCAX */
+ feature = ARM_FEATURE_V8_SHA3;
+ break;
+ case 2: /* SM3SS1 */
+ feature = ARM_FEATURE_V8_SM3;
+ break;
+ default:
+ unallocated_encoding(s);
+ return;
+ }
+
+ if (!arm_dc_feature(s, feature)) {
+ unallocated_encoding(s);
+ return;
+ }
+
+ if (!fp_access_check(s)) {
+ return;
+ }
+
+ if (op0 < 2) {
+ TCGv_i64 tcg_op1, tcg_op2, tcg_op3, tcg_res[2];
+ int pass;
+
+ tcg_op1 = tcg_temp_new_i64();
+ tcg_op2 = tcg_temp_new_i64();
+ tcg_op3 = tcg_temp_new_i64();
+ tcg_res[0] = tcg_temp_new_i64();
+ tcg_res[1] = tcg_temp_new_i64();
+
+ for (pass = 0; pass < 2; pass++) {
+ read_vec_element(s, tcg_op1, rn, pass, MO_64);
+ read_vec_element(s, tcg_op2, rm, pass, MO_64);
+ read_vec_element(s, tcg_op3, ra, pass, MO_64);
+
+ if (op0 == 0) {
+ /* EOR3 */
+ tcg_gen_xor_i64(tcg_res[pass], tcg_op2, tcg_op3);
+ } else {
+ /* BCAX */
+ tcg_gen_andc_i64(tcg_res[pass], tcg_op2, tcg_op3);
+ }
+ tcg_gen_xor_i64(tcg_res[pass], tcg_res[pass], tcg_op1);
+ }
+ write_vec_element(s, tcg_res[0], rd, 0, MO_64);
+ write_vec_element(s, tcg_res[1], rd, 1, MO_64);
+
+ tcg_temp_free_i64(tcg_op1);
+ tcg_temp_free_i64(tcg_op2);
+ tcg_temp_free_i64(tcg_op3);
+ tcg_temp_free_i64(tcg_res[0]);
+ tcg_temp_free_i64(tcg_res[1]);
+ } else {
+ TCGv_i32 tcg_op1, tcg_op2, tcg_op3, tcg_res, tcg_zero;
+
+ tcg_op1 = tcg_temp_new_i32();
+ tcg_op2 = tcg_temp_new_i32();
+ tcg_op3 = tcg_temp_new_i32();
+ tcg_res = tcg_temp_new_i32();
+ tcg_zero = tcg_const_i32(0);
+
+ read_vec_element_i32(s, tcg_op1, rn, 3, MO_32);
+ read_vec_element_i32(s, tcg_op2, rm, 3, MO_32);
+ read_vec_element_i32(s, tcg_op3, ra, 3, MO_32);
+
+ tcg_gen_rotri_i32(tcg_res, tcg_op1, 20);
+ tcg_gen_add_i32(tcg_res, tcg_res, tcg_op2);
+ tcg_gen_add_i32(tcg_res, tcg_res, tcg_op3);
+ tcg_gen_rotri_i32(tcg_res, tcg_res, 25);
+
+ write_vec_element_i32(s, tcg_zero, rd, 0, MO_32);
+ write_vec_element_i32(s, tcg_zero, rd, 1, MO_32);
+ write_vec_element_i32(s, tcg_zero, rd, 2, MO_32);
+ write_vec_element_i32(s, tcg_res, rd, 3, MO_32);
+
+ tcg_temp_free_i32(tcg_op1);
+ tcg_temp_free_i32(tcg_op2);
+ tcg_temp_free_i32(tcg_op3);
+ tcg_temp_free_i32(tcg_res);
+ tcg_temp_free_i32(tcg_zero);
+ }
+}
+
+/* Crypto XAR
+ * 31 21 20 16 15 10 9 5 4 0
+ * +-----------------------+------+--------+------+------+
+ * | 1 1 0 0 1 1 1 0 1 0 0 | Rm | imm6 | Rn | Rd |
+ * +-----------------------+------+--------+------+------+
+ */
+static void disas_crypto_xar(DisasContext *s, uint32_t insn)
+{
+ int rm = extract32(insn, 16, 5);
+ int imm6 = extract32(insn, 10, 6);
+ int rn = extract32(insn, 5, 5);
+ int rd = extract32(insn, 0, 5);
+ TCGv_i64 tcg_op1, tcg_op2, tcg_res[2];
+ int pass;
+
+ if (!arm_dc_feature(s, ARM_FEATURE_V8_SHA3)) {
+ unallocated_encoding(s);
+ return;
+ }
+
+ if (!fp_access_check(s)) {
+ return;
+ }
+
+ tcg_op1 = tcg_temp_new_i64();
+ tcg_op2 = tcg_temp_new_i64();
+ tcg_res[0] = tcg_temp_new_i64();
+ tcg_res[1] = tcg_temp_new_i64();
+
+ for (pass = 0; pass < 2; pass++) {
+ read_vec_element(s, tcg_op1, rn, pass, MO_64);
+ read_vec_element(s, tcg_op2, rm, pass, MO_64);
+
+ tcg_gen_xor_i64(tcg_res[pass], tcg_op1, tcg_op2);
+ tcg_gen_rotri_i64(tcg_res[pass], tcg_res[pass], imm6);
+ }
+ write_vec_element(s, tcg_res[0], rd, 0, MO_64);
+ write_vec_element(s, tcg_res[1], rd, 1, MO_64);
+
+ tcg_temp_free_i64(tcg_op1);
+ tcg_temp_free_i64(tcg_op2);
+ tcg_temp_free_i64(tcg_res[0]);
+ tcg_temp_free_i64(tcg_res[1]);
+}
+
+/* Crypto three-reg imm2
+ * 31 21 20 16 15 14 13 12 11 10 9 5 4 0
+ * +-----------------------+------+-----+------+--------+------+------+
+ * | 1 1 0 0 1 1 1 0 0 1 0 | Rm | 1 0 | imm2 | opcode | Rn | Rd |
+ * +-----------------------+------+-----+------+--------+------+------+
+ */
+static void disas_crypto_three_reg_imm2(DisasContext *s, uint32_t insn)
+{
+ int opcode = extract32(insn, 10, 2);
+ int imm2 = extract32(insn, 12, 2);
+ int rm = extract32(insn, 16, 5);
+ int rn = extract32(insn, 5, 5);
+ int rd = extract32(insn, 0, 5);
+ TCGv_ptr tcg_rd_ptr, tcg_rn_ptr, tcg_rm_ptr;
+ TCGv_i32 tcg_imm2, tcg_opcode;
+
+ if (!arm_dc_feature(s, ARM_FEATURE_V8_SM3)) {
+ unallocated_encoding(s);
+ return;
+ }
+
+ if (!fp_access_check(s)) {
+ return;
+ }
+
+ tcg_rd_ptr = vec_full_reg_ptr(s, rd);
+ tcg_rn_ptr = vec_full_reg_ptr(s, rn);
+ tcg_rm_ptr = vec_full_reg_ptr(s, rm);
+ tcg_imm2 = tcg_const_i32(imm2);
+ tcg_opcode = tcg_const_i32(opcode);
+
+ gen_helper_crypto_sm3tt(tcg_rd_ptr, tcg_rn_ptr, tcg_rm_ptr, tcg_imm2,
+ tcg_opcode);
+
+ tcg_temp_free_ptr(tcg_rd_ptr);
+ tcg_temp_free_ptr(tcg_rn_ptr);
+ tcg_temp_free_ptr(tcg_rm_ptr);
+ tcg_temp_free_i32(tcg_imm2);
+ tcg_temp_free_i32(tcg_opcode);
+}
+
/* C3.6 Data processing - SIMD, inc Crypto
*
* As the decode gets a little complex we are using a table based
@@ -11161,6 +11951,11 @@ static const AArch64DecodeTable data_proc_simd[] = {
{ 0x4e280800, 0xff3e0c00, disas_crypto_aes },
{ 0x5e000000, 0xff208c00, disas_crypto_three_reg_sha },
{ 0x5e280800, 0xff3e0c00, disas_crypto_two_reg_sha },
+ { 0xce608000, 0xffe0b000, disas_crypto_three_reg_sha512 },
+ { 0xcec08000, 0xfffff000, disas_crypto_two_reg_sha512 },
+ { 0xce000000, 0xff808000, disas_crypto_four_reg },
+ { 0xce800000, 0xffe00000, disas_crypto_xar },
+ { 0xce408000, 0xffe0c000, disas_crypto_three_reg_imm2 },
{ 0x00000000, 0x00000000, NULL }
};
@@ -11263,6 +12058,8 @@ static int aarch64_tr_init_disas_context(DisasContextBase *dcbase,
dc->user = (dc->current_el == 0);
#endif
dc->fp_excp_el = ARM_TBFLAG_FPEXC_EL(dc->base.tb->flags);
+ dc->sve_excp_el = ARM_TBFLAG_SVEEXC_EL(dc->base.tb->flags);
+ dc->sve_len = (ARM_TBFLAG_ZCR_LEN(dc->base.tb->flags) + 1) * 16;
dc->vec_len = 0;
dc->vec_stride = 0;
dc->cp_regs = arm_cpu->cp_regs;
diff --git a/target/arm/translate.c b/target/arm/translate.c
index 55826b7e5a..1270022289 100644
--- a/target/arm/translate.c
+++ b/target/arm/translate.c
@@ -1512,13 +1512,12 @@ static inline void gen_vfp_st(DisasContext *s, int dp, TCGv_i32 addr)
}
}
-static inline long
-vfp_reg_offset (int dp, int reg)
+static inline long vfp_reg_offset(bool dp, unsigned reg)
{
if (dp) {
- return offsetof(CPUARMState, vfp.regs[reg]);
+ return offsetof(CPUARMState, vfp.zregs[reg >> 1].d[reg & 1]);
} else {
- long ofs = offsetof(CPUARMState, vfp.regs[reg >> 1]);
+ long ofs = offsetof(CPUARMState, vfp.zregs[reg >> 2].d[(reg >> 1) & 1]);
if (reg & 1) {
ofs += offsetof(CPU_DoubleU, l.upper);
} else {
@@ -9926,6 +9925,7 @@ static void disas_thumb2_insn(DisasContext *s, uint32_t insn)
tcg_temp_free_i32(addr);
tcg_temp_free_i32(op);
store_reg(s, rd, ttresp);
+ break;
}
goto illegal_op;
}
diff --git a/target/arm/translate.h b/target/arm/translate.h
index 3f4df91e5e..c47febf99d 100644
--- a/target/arm/translate.h
+++ b/target/arm/translate.h
@@ -29,6 +29,8 @@ typedef struct DisasContext {
bool tbi1; /* TBI1 for EL0/1, not used for EL2/3 */
bool ns; /* Use non-secure CPREG bank on access */
int fp_excp_el; /* FP exception EL or 0 if enabled */
+ int sve_excp_el; /* SVE exception EL or 0 if enabled */
+ int sve_len; /* SVE vector length in bytes */
/* Flag indicating that exceptions from secure mode are routed to EL3. */
bool secure_routed_to_el3;
bool vfp_enabled; /* FP enabled via FPSCR.EN */
diff --git a/target/s390x/cpu.c b/target/s390x/cpu.c
index 979469dc3c..da7cb9c278 100644
--- a/target/s390x/cpu.c
+++ b/target/s390x/cpu.c
@@ -100,7 +100,6 @@ static void s390_cpu_initial_reset(CPUState *s)
{
S390CPU *cpu = S390_CPU(s);
CPUS390XState *env = &cpu->env;
- int i;
s390_cpu_reset(s);
/* initial reset does not clear everything! */
@@ -116,10 +115,6 @@ static void s390_cpu_initial_reset(CPUState *s)
env->gbea = 1;
env->pfault_token = -1UL;
- for (i = 0; i < ARRAY_SIZE(env->io_index); i++) {
- env->io_index[i] = -1;
- }
- env->mchk_index = -1;
/* tininess for underflow is detected before rounding */
set_float_detect_tininess(float_tininess_before_rounding,
@@ -137,7 +132,6 @@ static void s390_cpu_full_reset(CPUState *s)
S390CPU *cpu = S390_CPU(s);
S390CPUClass *scc = S390_CPU_GET_CLASS(cpu);
CPUS390XState *env = &cpu->env;
- int i;
scc->parent_reset(s);
cpu->env.sigp_order = 0;
@@ -153,10 +147,6 @@ static void s390_cpu_full_reset(CPUState *s)
env->gbea = 1;
env->pfault_token = -1UL;
- for (i = 0; i < ARRAY_SIZE(env->io_index); i++) {
- env->io_index[i] = -1;
- }
- env->mchk_index = -1;
/* tininess for underflow is detected before rounding */
set_float_detect_tininess(float_tininess_before_rounding,
diff --git a/target/s390x/cpu.h b/target/s390x/cpu.h
index a1123ad621..21ce40d5b6 100644
--- a/target/s390x/cpu.h
+++ b/target/s390x/cpu.h
@@ -53,12 +53,6 @@
#define MMU_USER_IDX 0
-#define MAX_IO_QUEUE 16
-#define MAX_MCHK_QUEUE 16
-
-#define PSW_MCHK_MASK 0x0004000000000000
-#define PSW_IO_MASK 0x0200000000000000
-
#define S390_MAX_CPUS 248
typedef struct PSW {
@@ -66,17 +60,6 @@ typedef struct PSW {
uint64_t addr;
} PSW;
-typedef struct IOIntQueue {
- uint16_t id;
- uint16_t nr;
- uint32_t parm;
- uint32_t word;
-} IOIntQueue;
-
-typedef struct MchkQueue {
- uint16_t type;
-} MchkQueue;
-
struct CPUS390XState {
uint64_t regs[16]; /* GP registers */
/*
@@ -122,15 +105,9 @@ struct CPUS390XState {
uint64_t cregs[16]; /* control registers */
- IOIntQueue io_queue[MAX_IO_QUEUE][8];
- MchkQueue mchk_queue[MAX_MCHK_QUEUE];
-
int pending_int;
- uint32_t service_param;
uint16_t external_call_addr;
DECLARE_BITMAP(emergency_signals, S390_MAX_CPUS);
- int io_index[8];
- int mchk_index;
uint64_t ckc;
uint64_t cputm;
@@ -409,9 +386,6 @@ static inline void cpu_get_tb_cpu_state(CPUS390XState* env, target_ulong *pc,
#define EXCP_IO 7 /* I/O interrupt */
#define EXCP_MCHK 8 /* machine check */
-#define INTERRUPT_IO (1 << 0)
-#define INTERRUPT_MCHK (1 << 1)
-#define INTERRUPT_EXT_SERVICE (1 << 2)
#define INTERRUPT_EXT_CPU_TIMER (1 << 3)
#define INTERRUPT_EXT_CLOCK_COMPARATOR (1 << 4)
#define INTERRUPT_EXTERNAL_CALL (1 << 5)
@@ -452,62 +426,66 @@ static inline void setcc(S390CPU *cpu, uint64_t cc)
}
/* STSI */
-#define STSI_LEVEL_MASK 0x00000000f0000000ULL
-#define STSI_LEVEL_CURRENT 0x0000000000000000ULL
-#define STSI_LEVEL_1 0x0000000010000000ULL
-#define STSI_LEVEL_2 0x0000000020000000ULL
-#define STSI_LEVEL_3 0x0000000030000000ULL
+#define STSI_R0_FC_MASK 0x00000000f0000000ULL
+#define STSI_R0_FC_CURRENT 0x0000000000000000ULL
+#define STSI_R0_FC_LEVEL_1 0x0000000010000000ULL
+#define STSI_R0_FC_LEVEL_2 0x0000000020000000ULL
+#define STSI_R0_FC_LEVEL_3 0x0000000030000000ULL
#define STSI_R0_RESERVED_MASK 0x000000000fffff00ULL
#define STSI_R0_SEL1_MASK 0x00000000000000ffULL
#define STSI_R1_RESERVED_MASK 0x00000000ffff0000ULL
#define STSI_R1_SEL2_MASK 0x000000000000ffffULL
/* Basic Machine Configuration */
-struct sysib_111 {
- uint32_t res1[8];
+typedef struct SysIB_111 {
+ uint8_t res1[32];
uint8_t manuf[16];
uint8_t type[4];
uint8_t res2[12];
uint8_t model[16];
uint8_t sequence[16];
uint8_t plant[4];
- uint8_t res3[156];
-};
+ uint8_t res3[3996];
+} SysIB_111;
+QEMU_BUILD_BUG_ON(sizeof(SysIB_111) != 4096);
/* Basic Machine CPU */
-struct sysib_121 {
- uint32_t res1[80];
+typedef struct SysIB_121 {
+ uint8_t res1[80];
uint8_t sequence[16];
uint8_t plant[4];
uint8_t res2[2];
uint16_t cpu_addr;
- uint8_t res3[152];
-};
+ uint8_t res3[3992];
+} SysIB_121;
+QEMU_BUILD_BUG_ON(sizeof(SysIB_121) != 4096);
/* Basic Machine CPUs */
-struct sysib_122 {
+typedef struct SysIB_122 {
uint8_t res1[32];
uint32_t capability;
uint16_t total_cpus;
- uint16_t active_cpus;
+ uint16_t conf_cpus;
uint16_t standby_cpus;
uint16_t reserved_cpus;
uint16_t adjustments[2026];
-};
+} SysIB_122;
+QEMU_BUILD_BUG_ON(sizeof(SysIB_122) != 4096);
/* LPAR CPU */
-struct sysib_221 {
- uint32_t res1[80];
+typedef struct SysIB_221 {
+ uint8_t res1[80];
uint8_t sequence[16];
uint8_t plant[4];
uint16_t cpu_id;
uint16_t cpu_addr;
- uint8_t res3[152];
-};
+ uint8_t res3[3992];
+} SysIB_221;
+QEMU_BUILD_BUG_ON(sizeof(SysIB_221) != 4096);
/* LPAR CPUs */
-struct sysib_222 {
- uint32_t res1[32];
+typedef struct SysIB_222 {
+ uint8_t res1[32];
uint16_t lpar_num;
uint8_t res2;
uint8_t lcpuc;
@@ -520,11 +498,12 @@ struct sysib_222 {
uint8_t res3[16];
uint16_t dedicated_cpus;
uint16_t shared_cpus;
- uint8_t res4[180];
-};
+ uint8_t res4[4020];
+} SysIB_222;
+QEMU_BUILD_BUG_ON(sizeof(SysIB_222) != 4096);
/* VM CPUs */
-struct sysib_322 {
+typedef struct SysIB_322 {
uint8_t res1[31];
uint8_t count;
struct {
@@ -543,7 +522,18 @@ struct sysib_322 {
} vm[8];
uint8_t res4[1504];
uint8_t ext_names[8][256];
-};
+} SysIB_322;
+QEMU_BUILD_BUG_ON(sizeof(SysIB_322) != 4096);
+
+typedef union SysIB {
+ SysIB_111 sysib_111;
+ SysIB_121 sysib_121;
+ SysIB_122 sysib_122;
+ SysIB_221 sysib_221;
+ SysIB_222 sysib_222;
+ SysIB_322 sysib_322;
+} SysIB;
+QEMU_BUILD_BUG_ON(sizeof(SysIB) != 4096);
/* MMU defines */
#define _ASCE_ORIGIN ~0xfffULL /* segment table origin */
@@ -718,6 +708,10 @@ static inline unsigned int s390_cpu_set_state(uint8_t cpu_state, S390CPU *cpu)
return 0;
}
#endif /* CONFIG_USER_ONLY */
+static inline uint8_t s390_cpu_get_state(S390CPU *cpu)
+{
+ return cpu->env.cpu_state;
+}
/* cpu_models.c */
@@ -752,7 +746,6 @@ void s390_program_interrupt(CPUS390XState *env, uint32_t code, int ilen,
/* service interrupts are floating therefore we must not pass an cpustate */
void s390_sclp_extint(uint32_t parm);
-
/* mmu_helper.c */
int s390_cpu_virt_mem_rw(S390CPU *cpu, vaddr laddr, uint8_t ar, void *hostbuf,
int len, bool is_write);
diff --git a/target/s390x/cpu_features.c b/target/s390x/cpu_features.c
index 85d10b5710..a5619f2893 100644
--- a/target/s390x/cpu_features.c
+++ b/target/s390x/cpu_features.c
@@ -156,8 +156,12 @@ static const S390FeatDef s390_features[] = {
FEAT_INIT("ptff-qpc", S390_FEAT_TYPE_PTFF, 3, "PTFF Query Physical Clock"),
FEAT_INIT("ptff-qui", S390_FEAT_TYPE_PTFF, 4, "PTFF Query UTC Information"),
FEAT_INIT("ptff-qtou", S390_FEAT_TYPE_PTFF, 5, "PTFF Query TOD Offset User"),
+ FEAT_INIT("ptff-qsie", S390_FEAT_TYPE_PTFF, 10, "PTFF Query Steering Information Extended"),
+ FEAT_INIT("ptff-qtoue", S390_FEAT_TYPE_PTFF, 13, "PTFF Query TOD Offset User Extended"),
FEAT_INIT("ptff-sto", S390_FEAT_TYPE_PTFF, 65, "PTFF Set TOD Offset"),
FEAT_INIT("ptff-stou", S390_FEAT_TYPE_PTFF, 69, "PTFF Set TOD Offset User"),
+ FEAT_INIT("ptff-stoe", S390_FEAT_TYPE_PTFF, 73, "PTFF Set TOD Offset Extended"),
+ FEAT_INIT("ptff-stoue", S390_FEAT_TYPE_PTFF, 77, "PTFF Set TOD Offset User Extended"),
FEAT_INIT("kmac-dea", S390_FEAT_TYPE_KMAC, 1, "KMAC DEA"),
FEAT_INIT("kmac-tdea-128", S390_FEAT_TYPE_KMAC, 2, "KMAC TDEA-128"),
@@ -445,6 +449,7 @@ static S390FeatGroupDef s390_feature_groups[] = {
FEAT_GROUP_INIT("plo", PLO, "Perform-locked-operation facility"),
FEAT_GROUP_INIT("tods", TOD_CLOCK_STEERING, "Tod-clock-steering facility"),
FEAT_GROUP_INIT("gen13ptff", GEN13_PTFF, "PTFF enhancements introduced with z13"),
+ FEAT_GROUP_INIT("mepochptff", MULTIPLE_EPOCH_PTFF, "PTFF enhancements introduced with Multiple-epoch facility"),
FEAT_GROUP_INIT("msa", MSA, "Message-security-assist facility"),
FEAT_GROUP_INIT("msa1", MSA_EXT_1, "Message-security-assist-extension 1 facility"),
FEAT_GROUP_INIT("msa2", MSA_EXT_2, "Message-security-assist-extension 2 facility"),
diff --git a/target/s390x/cpu_features_def.h b/target/s390x/cpu_features_def.h
index 4d930871b4..7c5915c7b2 100644
--- a/target/s390x/cpu_features_def.h
+++ b/target/s390x/cpu_features_def.h
@@ -151,8 +151,12 @@ typedef enum {
S390_FEAT_PTFF_QPT,
S390_FEAT_PTFF_QUI,
S390_FEAT_PTFF_QTOU,
+ S390_FEAT_PTFF_QSIE,
+ S390_FEAT_PTFF_QTOUE,
S390_FEAT_PTFF_STO,
S390_FEAT_PTFF_STOU,
+ S390_FEAT_PTFF_STOE,
+ S390_FEAT_PTFF_STOUE,
/* KMAC */
S390_FEAT_KMAC_DEA,
diff --git a/target/s390x/cpu_models.c b/target/s390x/cpu_models.c
index 584c409a19..1d5f0da4fe 100644
--- a/target/s390x/cpu_models.c
+++ b/target/s390x/cpu_models.c
@@ -23,6 +23,7 @@
#include "qapi/qmp/qdict.h"
#ifndef CONFIG_USER_ONLY
#include "sysemu/arch_init.h"
+#include "hw/pci/pci.h"
#endif
#define CPUDEF_INIT(_type, _gen, _ec_ga, _mha_pow, _hmfai, _name, _desc) \
@@ -1271,6 +1272,11 @@ static void register_types(void)
/* init all bitmaps from gnerated data initially */
s390_init_feat_bitmap(qemu_max_cpu_feat_init, qemu_max_cpu_feat);
+#ifndef CONFIG_USER_ONLY
+ if (!pci_available) {
+ clear_bit(S390_FEAT_ZPCI, qemu_max_cpu_feat);
+ }
+#endif
for (i = 0; i < ARRAY_SIZE(s390_cpu_defs); i++) {
s390_init_feat_bitmap(s390_cpu_defs[i].base_init,
s390_cpu_defs[i].base_feat);
diff --git a/target/s390x/excp_helper.c b/target/s390x/excp_helper.c
index 6967fbfa25..411051edc3 100644
--- a/target/s390x/excp_helper.c
+++ b/target/s390x/excp_helper.c
@@ -28,6 +28,7 @@
#include "exec/address-spaces.h"
#ifndef CONFIG_USER_ONLY
#include "sysemu/sysemu.h"
+#include "hw/s390x/s390_flic.h"
#endif
/* #define DEBUG_S390 */
@@ -236,6 +237,7 @@ static void do_svc_interrupt(CPUS390XState *env)
static void do_ext_interrupt(CPUS390XState *env)
{
+ QEMUS390FLICState *flic = QEMU_S390_FLIC(s390_get_flic());
S390CPU *cpu = s390_env_get_cpu(env);
uint64_t mask, addr;
uint16_t cpu_addr;
@@ -272,17 +274,14 @@ static void do_ext_interrupt(CPUS390XState *env)
lowcore->ext_int_code = cpu_to_be16(EXT_CPU_TIMER);
lowcore->cpu_addr = 0;
env->pending_int &= ~INTERRUPT_EXT_CPU_TIMER;
- } else if ((env->pending_int & INTERRUPT_EXT_SERVICE) &&
+ } else if (qemu_s390_flic_has_service(flic) &&
(env->cregs[0] & CR0_SERVICE_SC)) {
- /*
- * FIXME: floating IRQs should be considered by all CPUs and
- * shuld not get cleared by CPU reset.
- */
+ uint32_t param;
+
+ param = qemu_s390_flic_dequeue_service(flic);
lowcore->ext_int_code = cpu_to_be16(EXT_SERVICE);
- lowcore->ext_params = cpu_to_be32(env->service_param);
+ lowcore->ext_params = cpu_to_be32(param);
lowcore->cpu_addr = 0;
- env->service_param = 0;
- env->pending_int &= ~INTERRUPT_EXT_SERVICE;
} else {
g_assert_not_reached();
}
@@ -302,95 +301,46 @@ static void do_ext_interrupt(CPUS390XState *env)
static void do_io_interrupt(CPUS390XState *env)
{
- S390CPU *cpu = s390_env_get_cpu(env);
+ QEMUS390FLICState *flic = QEMU_S390_FLIC(s390_get_flic());
+ uint64_t mask, addr;
+ QEMUS390FlicIO *io;
LowCore *lowcore;
- IOIntQueue *q;
- uint8_t isc;
- int disable = 1;
- int found = 0;
-
- if (!(env->psw.mask & PSW_MASK_IO)) {
- cpu_abort(CPU(cpu), "I/O int w/o I/O mask\n");
- }
-
- for (isc = 0; isc < ARRAY_SIZE(env->io_index); isc++) {
- uint64_t isc_bits;
-
- if (env->io_index[isc] < 0) {
- continue;
- }
- if (env->io_index[isc] >= MAX_IO_QUEUE) {
- cpu_abort(CPU(cpu), "I/O queue overrun for isc %d: %d\n",
- isc, env->io_index[isc]);
- }
-
- q = &env->io_queue[env->io_index[isc]][isc];
- isc_bits = ISC_TO_ISC_BITS(IO_INT_WORD_ISC(q->word));
- if (!(env->cregs[6] & isc_bits)) {
- disable = 0;
- continue;
- }
- if (!found) {
- uint64_t mask, addr;
-
- found = 1;
- lowcore = cpu_map_lowcore(env);
-
- lowcore->subchannel_id = cpu_to_be16(q->id);
- lowcore->subchannel_nr = cpu_to_be16(q->nr);
- lowcore->io_int_parm = cpu_to_be32(q->parm);
- lowcore->io_int_word = cpu_to_be32(q->word);
- lowcore->io_old_psw.mask = cpu_to_be64(get_psw_mask(env));
- lowcore->io_old_psw.addr = cpu_to_be64(env->psw.addr);
- mask = be64_to_cpu(lowcore->io_new_psw.mask);
- addr = be64_to_cpu(lowcore->io_new_psw.addr);
- cpu_unmap_lowcore(lowcore);
+ g_assert(env->psw.mask & PSW_MASK_IO);
+ io = qemu_s390_flic_dequeue_io(flic, env->cregs[6]);
+ g_assert(io);
- env->io_index[isc]--;
+ lowcore = cpu_map_lowcore(env);
- DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__,
- env->psw.mask, env->psw.addr);
- load_psw(env, mask, addr);
- }
- if (env->io_index[isc] >= 0) {
- disable = 0;
- }
- continue;
- }
+ lowcore->subchannel_id = cpu_to_be16(io->id);
+ lowcore->subchannel_nr = cpu_to_be16(io->nr);
+ lowcore->io_int_parm = cpu_to_be32(io->parm);
+ lowcore->io_int_word = cpu_to_be32(io->word);
+ lowcore->io_old_psw.mask = cpu_to_be64(get_psw_mask(env));
+ lowcore->io_old_psw.addr = cpu_to_be64(env->psw.addr);
+ mask = be64_to_cpu(lowcore->io_new_psw.mask);
+ addr = be64_to_cpu(lowcore->io_new_psw.addr);
- if (disable) {
- env->pending_int &= ~INTERRUPT_IO;
- }
+ cpu_unmap_lowcore(lowcore);
+ g_free(io);
+ DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__, env->psw.mask,
+ env->psw.addr);
+ load_psw(env, mask, addr);
}
static void do_mchk_interrupt(CPUS390XState *env)
{
- S390CPU *cpu = s390_env_get_cpu(env);
+ QEMUS390FLICState *flic = QEMU_S390_FLIC(s390_get_flic());
uint64_t mask, addr;
LowCore *lowcore;
- MchkQueue *q;
int i;
- if (!(env->psw.mask & PSW_MASK_MCHECK)) {
- cpu_abort(CPU(cpu), "Machine check w/o mchk mask\n");
- }
-
- if (env->mchk_index < 0 || env->mchk_index >= MAX_MCHK_QUEUE) {
- cpu_abort(CPU(cpu), "Mchk queue overrun: %d\n", env->mchk_index);
- }
-
- q = &env->mchk_queue[env->mchk_index];
+ /* for now we only support channel report machine checks (floating) */
+ g_assert(env->psw.mask & PSW_MASK_MCHECK);
+ g_assert(env->cregs[14] & CR14_CHANNEL_REPORT_SC);
- if (q->type != 1) {
- /* Don't know how to handle this... */
- cpu_abort(CPU(cpu), "Unknown machine check type %d\n", q->type);
- }
- if (!(env->cregs[14] & (1 << 28))) {
- /* CRW machine checks disabled */
- return;
- }
+ qemu_s390_flic_dequeue_crw_mchk(flic);
lowcore = cpu_map_lowcore(env);
@@ -417,11 +367,6 @@ static void do_mchk_interrupt(CPUS390XState *env)
cpu_unmap_lowcore(lowcore);
- env->mchk_index--;
- if (env->mchk_index == -1) {
- env->pending_int &= ~INTERRUPT_MCHK;
- }
-
DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__,
env->psw.mask, env->psw.addr);
@@ -430,12 +375,15 @@ static void do_mchk_interrupt(CPUS390XState *env)
void s390_cpu_do_interrupt(CPUState *cs)
{
+ QEMUS390FLICState *flic = QEMU_S390_FLIC(s390_get_flic());
S390CPU *cpu = S390_CPU(cs);
CPUS390XState *env = &cpu->env;
+ bool stopped = false;
qemu_log_mask(CPU_LOG_INT, "%s: %d at pc=%" PRIx64 "\n",
__func__, cs->exception_index, env->psw.addr);
+try_deliver:
/* handle machine checks */
if (cs->exception_index == -1 && s390_cpu_has_mcck_int(cpu)) {
cs->exception_index = EXCP_MCHK;
@@ -478,20 +426,30 @@ void s390_cpu_do_interrupt(CPUState *cs)
break;
case EXCP_STOP:
do_stop_interrupt(env);
+ stopped = true;
break;
}
- /* WAIT PSW during interrupt injection or STOP interrupt */
- if (cs->exception_index == EXCP_HLT) {
- /* don't trigger a cpu_loop_exit(), use an interrupt instead */
- cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HALT);
+ if (cs->exception_index != -1 && !stopped) {
+ /* check if there are more pending interrupts to deliver */
+ cs->exception_index = -1;
+ goto try_deliver;
}
cs->exception_index = -1;
/* we might still have pending interrupts, but not deliverable */
- if (!env->pending_int) {
+ if (!env->pending_int && !qemu_s390_flic_has_any(flic)) {
cs->interrupt_request &= ~CPU_INTERRUPT_HARD;
}
+
+ /* WAIT PSW during interrupt injection or STOP interrupt */
+ if ((env->psw.mask & PSW_MASK_WAIT) || stopped) {
+ /* don't trigger a cpu_loop_exit(), use an interrupt instead */
+ cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HALT);
+ } else if (cs->halted) {
+ /* unhalt if we had a WAIT PSW somehwere in our injection chain */
+ s390_cpu_unhalt(cpu);
+ }
}
bool s390_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
@@ -509,6 +467,11 @@ bool s390_cpu_exec_interrupt(CPUState *cs, int interrupt_request)
s390_cpu_do_interrupt(cs);
return true;
}
+ if (env->psw.mask & PSW_MASK_WAIT) {
+ /* Woken up because of a floating interrupt but it has already
+ * been delivered. Go back to sleep. */
+ cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HALT);
+ }
}
return false;
}
diff --git a/target/s390x/gen-features.c b/target/s390x/gen-features.c
index bd483333ae..0cdbc15378 100644
--- a/target/s390x/gen-features.c
+++ b/target/s390x/gen-features.c
@@ -57,6 +57,12 @@
S390_FEAT_PTFF_QTOU, \
S390_FEAT_PTFF_STOU
+#define S390_FEAT_GROUP_MULTIPLE_EPOCH_PTFF \
+ S390_FEAT_PTFF_QSIE, \
+ S390_FEAT_PTFF_QTOUE, \
+ S390_FEAT_PTFF_STOE, \
+ S390_FEAT_PTFF_STOUE
+
#define S390_FEAT_GROUP_MSA \
S390_FEAT_MSA, \
S390_FEAT_KMAC_DEA, \
@@ -217,6 +223,9 @@ static uint16_t group_TOD_CLOCK_STEERING[] = {
static uint16_t group_GEN13_PTFF[] = {
S390_FEAT_GROUP_GEN13_PTFF,
};
+static uint16_t group_MULTIPLE_EPOCH_PTFF[] = {
+ S390_FEAT_GROUP_MULTIPLE_EPOCH_PTFF,
+};
static uint16_t group_MSA[] = {
S390_FEAT_GROUP_MSA,
};
@@ -464,6 +473,7 @@ static uint16_t full_GEN14_GA1[] = {
S390_FEAT_CMM_NT,
S390_FEAT_HPMA2,
S390_FEAT_SIE_KSS,
+ S390_FEAT_GROUP_MULTIPLE_EPOCH_PTFF,
};
/* Default features (in order of release)
@@ -570,8 +580,10 @@ static uint16_t qemu_LATEST[] = {
S390_FEAT_STFLE_49,
S390_FEAT_LOCAL_TLB_CLEARING,
S390_FEAT_INTERLOCKED_ACCESS_2,
- S390_FEAT_MSA_EXT_4,
+ S390_FEAT_ADAPTER_EVENT_NOTIFICATION,
+ S390_FEAT_ADAPTER_INT_SUPPRESSION,
S390_FEAT_MSA_EXT_3,
+ S390_FEAT_MSA_EXT_4,
};
/* add all new definitions before this point */
@@ -580,6 +592,8 @@ static uint16_t qemu_MAX[] = {
S390_FEAT_STFLE_53,
/* generates a dependency warning, leave it out for now */
S390_FEAT_MSA_EXT_5,
+ /* only with CONFIG_PCI */
+ S390_FEAT_ZPCI,
};
/****** END FEATURE DEFS ******/
@@ -662,6 +676,7 @@ static FeatGroupDefSpec FeatGroupDef[] = {
FEAT_GROUP_INITIALIZER(PLO),
FEAT_GROUP_INITIALIZER(TOD_CLOCK_STEERING),
FEAT_GROUP_INITIALIZER(GEN13_PTFF),
+ FEAT_GROUP_INITIALIZER(MULTIPLE_EPOCH_PTFF),
FEAT_GROUP_INITIALIZER(MSA),
FEAT_GROUP_INITIALIZER(MSA_EXT_1),
FEAT_GROUP_INITIALIZER(MSA_EXT_2),
diff --git a/target/s390x/helper.h b/target/s390x/helper.h
index 59a1d9869b..59cba86a27 100644
--- a/target/s390x/helper.h
+++ b/target/s390x/helper.h
@@ -170,6 +170,16 @@ DEF_HELPER_4(schm, void, env, i64, i64, i64)
DEF_HELPER_3(ssch, void, env, i64, i64)
DEF_HELPER_2(stcrw, void, env, i64)
DEF_HELPER_3(stsch, void, env, i64, i64)
+DEF_HELPER_2(tpi, i32, env, i64)
DEF_HELPER_3(tsch, void, env, i64, i64)
DEF_HELPER_2(chsc, void, env, i64)
+
+DEF_HELPER_2(clp, void, env, i32)
+DEF_HELPER_3(pcilg, void, env, i32, i32)
+DEF_HELPER_3(pcistg, void, env, i32, i32)
+DEF_HELPER_4(stpcifc, void, env, i32, i64, i32)
+DEF_HELPER_3(sic, void, env, i64, i64)
+DEF_HELPER_3(rpcit, void, env, i32, i32)
+DEF_HELPER_5(pcistb, void, env, i32, i32, i64, i32)
+DEF_HELPER_4(mpcifc, void, env, i32, i64, i32)
#endif
diff --git a/target/s390x/insn-data.def b/target/s390x/insn-data.def
index 11ee43dcbc..621e10d615 100644
--- a/target/s390x/insn-data.def
+++ b/target/s390x/insn-data.def
@@ -1063,8 +1063,22 @@
C(0xb233, SSCH, S, Z, 0, insn, 0, 0, ssch, 0)
C(0xb239, STCRW, S, Z, 0, insn, 0, 0, stcrw, 0)
C(0xb234, STSCH, S, Z, 0, insn, 0, 0, stsch, 0)
+ C(0xb236, TPI , S, Z, la2, 0, 0, 0, tpi, 0)
C(0xb235, TSCH, S, Z, 0, insn, 0, 0, tsch, 0)
/* ??? Not listed in PoO ninth edition, but there's a linux driver that
uses it: "A CHSC subchannel is usually present on LPAR only." */
C(0xb25f, CHSC, RRE, Z, 0, insn, 0, 0, chsc, 0)
+
+/* zPCI Instructions */
+ /* None of these instructions are documented in the PoP, so this is all
+ based upon target/s390x/kvm.c and Linux code and likely incomplete */
+ C(0xebd0, PCISTB, RSY_a, PCI, la2, 0, 0, 0, pcistb, 0)
+ C(0xebd1, SIC, RSY_a, AIS, r1, r3, 0, 0, sic, 0)
+ C(0xb9a0, CLP, RRF_c, PCI, 0, 0, 0, 0, clp, 0)
+ C(0xb9d0, PCISTG, RRE, PCI, 0, 0, 0, 0, pcistg, 0)
+ C(0xb9d2, PCILG, RRE, PCI, 0, 0, 0, 0, pcilg, 0)
+ C(0xb9d3, RPCIT, RRE, PCI, 0, 0, 0, 0, rpcit, 0)
+ C(0xe3d0, MPCIFC, RXY_a, PCI, la2, 0, 0, 0, mpcifc, 0)
+ C(0xe3d4, STPCIFC, RXY_a, PCI, la2, 0, 0, 0, stpcifc, 0)
+
#endif /* CONFIG_USER_ONLY */
diff --git a/target/s390x/internal.h b/target/s390x/internal.h
index fea165ffe4..d911e84958 100644
--- a/target/s390x/internal.h
+++ b/target/s390x/internal.h
@@ -278,11 +278,6 @@ static inline void s390_do_cpu_full_reset(CPUState *cs, run_on_cpu_data arg)
cpu_reset(cs);
}
-static inline uint8_t s390_cpu_get_state(S390CPU *cpu)
-{
- return cpu->env.cpu_state;
-}
-
/* arch_dump.c */
int s390_cpu_write_elf64_note(WriteCoreDumpFunction f, CPUState *cs,
diff --git a/target/s390x/interrupt.c b/target/s390x/interrupt.c
index 39c026b8b5..25cfb3eef8 100644
--- a/target/s390x/interrupt.c
+++ b/target/s390x/interrupt.c
@@ -15,6 +15,9 @@
#include "exec/exec-all.h"
#include "sysemu/kvm.h"
#include "hw/s390x/ioinst.h"
+#if !defined(CONFIG_USER_ONLY)
+#include "hw/s390x/s390_flic.h"
+#endif
/* Ensure to exit the TB after this call! */
void trigger_pgm_exception(CPUS390XState *env, uint32_t code, uint32_t ilen)
@@ -55,17 +58,6 @@ void s390_program_interrupt(CPUS390XState *env, uint32_t code, int ilen,
}
#if !defined(CONFIG_USER_ONLY)
-static void cpu_inject_service(S390CPU *cpu, uint32_t param)
-{
- CPUS390XState *env = &cpu->env;
-
- /* multiplexing is good enough for sclp - kvm does it internally as well*/
- env->service_param |= param;
-
- env->pending_int |= INTERRUPT_EXT_SERVICE;
- cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
-}
-
void cpu_inject_clock_comparator(S390CPU *cpu)
{
CPUS390XState *env = &cpu->env;
@@ -134,48 +126,6 @@ void cpu_inject_stop(S390CPU *cpu)
cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
}
-static void cpu_inject_io(S390CPU *cpu, uint16_t subchannel_id,
- uint16_t subchannel_number,
- uint32_t io_int_parm, uint32_t io_int_word)
-{
- CPUS390XState *env = &cpu->env;
- int isc = IO_INT_WORD_ISC(io_int_word);
-
- if (env->io_index[isc] == MAX_IO_QUEUE - 1) {
- /* ugh - can't queue anymore. Let's drop. */
- return;
- }
-
- env->io_index[isc]++;
- assert(env->io_index[isc] < MAX_IO_QUEUE);
-
- env->io_queue[env->io_index[isc]][isc].id = subchannel_id;
- env->io_queue[env->io_index[isc]][isc].nr = subchannel_number;
- env->io_queue[env->io_index[isc]][isc].parm = io_int_parm;
- env->io_queue[env->io_index[isc]][isc].word = io_int_word;
-
- env->pending_int |= INTERRUPT_IO;
- cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
-}
-
-static void cpu_inject_crw_mchk(S390CPU *cpu)
-{
- CPUS390XState *env = &cpu->env;
-
- if (env->mchk_index == MAX_MCHK_QUEUE - 1) {
- /* ugh - can't queue anymore. Let's drop. */
- return;
- }
-
- env->mchk_index++;
- assert(env->mchk_index < MAX_MCHK_QUEUE);
-
- env->mchk_queue[env->mchk_index].type = 1;
-
- env->pending_int |= INTERRUPT_MCHK;
- cpu_interrupt(CPU(cpu), CPU_INTERRUPT_HARD);
-}
-
/*
* All of the following interrupts are floating, i.e. not per-vcpu.
* We just need a dummy cpustate in order to be able to inject in the
@@ -183,53 +133,50 @@ static void cpu_inject_crw_mchk(S390CPU *cpu)
*/
void s390_sclp_extint(uint32_t parm)
{
- if (kvm_enabled()) {
- kvm_s390_service_interrupt(parm);
- } else {
- S390CPU *dummy_cpu = s390_cpu_addr2state(0);
+ S390FLICState *fs = s390_get_flic();
+ S390FLICStateClass *fsc = s390_get_flic_class(fs);
- cpu_inject_service(dummy_cpu, parm);
- }
+ fsc->inject_service(fs, parm);
}
void s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr,
uint32_t io_int_parm, uint32_t io_int_word)
{
- if (kvm_enabled()) {
- kvm_s390_io_interrupt(subchannel_id, subchannel_nr, io_int_parm,
- io_int_word);
- } else {
- S390CPU *dummy_cpu = s390_cpu_addr2state(0);
+ S390FLICState *fs = s390_get_flic();
+ S390FLICStateClass *fsc = s390_get_flic_class(fs);
- cpu_inject_io(dummy_cpu, subchannel_id, subchannel_nr, io_int_parm,
- io_int_word);
- }
+ fsc->inject_io(fs, subchannel_id, subchannel_nr, io_int_parm, io_int_word);
}
void s390_crw_mchk(void)
{
- if (kvm_enabled()) {
- kvm_s390_crw_mchk();
- } else {
- S390CPU *dummy_cpu = s390_cpu_addr2state(0);
+ S390FLICState *fs = s390_get_flic();
+ S390FLICStateClass *fsc = s390_get_flic_class(fs);
- cpu_inject_crw_mchk(dummy_cpu);
- }
+ fsc->inject_crw_mchk(fs);
}
bool s390_cpu_has_mcck_int(S390CPU *cpu)
{
+ QEMUS390FLICState *flic = s390_get_qemu_flic(s390_get_flic());
CPUS390XState *env = &cpu->env;
if (!(env->psw.mask & PSW_MASK_MCHECK)) {
return false;
}
- return env->pending_int & INTERRUPT_MCHK;
+ /* for now we only support channel report machine checks (floating) */
+ if (qemu_s390_flic_has_crw_mchk(flic) &&
+ (env->cregs[14] & CR14_CHANNEL_REPORT_SC)) {
+ return true;
+ }
+
+ return false;
}
bool s390_cpu_has_ext_int(S390CPU *cpu)
{
+ QEMUS390FLICState *flic = s390_get_qemu_flic(s390_get_flic());
CPUS390XState *env = &cpu->env;
if (!(env->psw.mask & PSW_MASK_EXT)) {
@@ -261,7 +208,7 @@ bool s390_cpu_has_ext_int(S390CPU *cpu)
return true;
}
- if ((env->pending_int & INTERRUPT_EXT_SERVICE) &&
+ if (qemu_s390_flic_has_service(flic) &&
(env->cregs[0] & CR0_SERVICE_SC)) {
return true;
}
@@ -271,13 +218,14 @@ bool s390_cpu_has_ext_int(S390CPU *cpu)
bool s390_cpu_has_io_int(S390CPU *cpu)
{
+ QEMUS390FLICState *flic = s390_get_qemu_flic(s390_get_flic());
CPUS390XState *env = &cpu->env;
if (!(env->psw.mask & PSW_MASK_IO)) {
return false;
}
- return env->pending_int & INTERRUPT_IO;
+ return qemu_s390_flic_has_io(flic, env->cregs[6]);
}
bool s390_cpu_has_restart_int(S390CPU *cpu)
diff --git a/target/s390x/kvm-stub.c b/target/s390x/kvm-stub.c
index 6bae3e99d3..8cdcf83845 100644
--- a/target/s390x/kvm-stub.c
+++ b/target/s390x/kvm-stub.c
@@ -12,10 +12,6 @@
#include "cpu.h"
#include "kvm_s390x.h"
-void kvm_s390_service_interrupt(uint32_t parm)
-{
-}
-
void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code)
{
}
@@ -30,15 +26,6 @@ void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code)
{
}
-void kvm_s390_io_interrupt(uint16_t subchannel_id, uint16_t subchannel_nr,
- uint32_t io_int_parm, uint32_t io_int_word)
-{
-}
-
-void kvm_s390_crw_mchk(void)
-{
-}
-
int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state)
{
return -ENOSYS;
diff --git a/target/s390x/kvm.c b/target/s390x/kvm.c
index 8fbbf72e4b..0301e9d519 100644
--- a/target/s390x/kvm.c
+++ b/target/s390x/kvm.c
@@ -1034,7 +1034,7 @@ void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq)
inject_vcpu_irq_legacy(cs, irq);
}
-static void __kvm_s390_floating_interrupt(struct kvm_s390_irq *irq)
+void kvm_s390_floating_interrupt_legacy(struct kvm_s390_irq *irq)
{
struct kvm_s390_interrupt kvmint = {};
int r;
@@ -1052,33 +1052,6 @@ static void __kvm_s390_floating_interrupt(struct kvm_s390_irq *irq)
}
}
-void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq)
-{
- static bool use_flic = true;
- int r;
-
- if (use_flic) {
- r = kvm_s390_inject_flic(irq);
- if (r == -ENOSYS) {
- use_flic = false;
- }
- if (!r) {
- return;
- }
- }
- __kvm_s390_floating_interrupt(irq);
-}
-
-void kvm_s390_service_interrupt(uint32_t parm)
-{
- struct kvm_s390_irq irq = {
- .type = KVM_S390_INT_SERVICE,
- .u.ext.ext_params = parm,
- };
-
- kvm_s390_floating_interrupt(&irq);
-}
-
void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code)
{
struct kvm_s390_irq irq = {
@@ -1690,10 +1663,10 @@ static int handle_tsch(S390CPU *cpu)
* If an I/O interrupt had been dequeued, we have to reinject it.
*/
if (run->s390_tsch.dequeued) {
- kvm_s390_io_interrupt(run->s390_tsch.subchannel_id,
- run->s390_tsch.subchannel_nr,
- run->s390_tsch.io_int_parm,
- run->s390_tsch.io_int_word);
+ s390_io_interrupt(run->s390_tsch.subchannel_id,
+ run->s390_tsch.subchannel_nr,
+ run->s390_tsch.io_int_parm,
+ run->s390_tsch.io_int_word);
}
ret = 0;
}
@@ -1702,7 +1675,7 @@ static int handle_tsch(S390CPU *cpu)
static void insert_stsi_3_2_2(S390CPU *cpu, __u64 addr, uint8_t ar)
{
- struct sysib_322 sysib;
+ SysIB_322 sysib;
int del;
if (s390_cpu_virt_mem_read(cpu, addr, ar, &sysib, sizeof(sysib))) {
@@ -1840,37 +1813,6 @@ bool kvm_arch_stop_on_emulation_error(CPUState *cpu)
return true;
}
-void kvm_s390_io_interrupt(uint16_t subchannel_id,
- uint16_t subchannel_nr, uint32_t io_int_parm,
- uint32_t io_int_word)
-{
- struct kvm_s390_irq irq = {
- .u.io.subchannel_id = subchannel_id,
- .u.io.subchannel_nr = subchannel_nr,
- .u.io.io_int_parm = io_int_parm,
- .u.io.io_int_word = io_int_word,
- };
-
- if (io_int_word & IO_INT_WORD_AI) {
- irq.type = KVM_S390_INT_IO(1, 0, 0, 0);
- } else {
- irq.type = KVM_S390_INT_IO(0, (subchannel_id & 0xff00) >> 8,
- (subchannel_id & 0x0006),
- subchannel_nr);
- }
- kvm_s390_floating_interrupt(&irq);
-}
-
-void kvm_s390_crw_mchk(void)
-{
- struct kvm_s390_irq irq = {
- .type = KVM_S390_MCHK,
- .u.mchk.cr14 = CR14_CHANNEL_REPORT_SC,
- .u.mchk.mcic = s390_build_validity_mcic() | MCIC_SC_CP,
- };
- kvm_s390_floating_interrupt(&irq);
-}
-
void kvm_s390_enable_css_support(S390CPU *cpu)
{
int r;
@@ -2279,6 +2221,14 @@ void kvm_s390_get_host_cpu_model(S390CPUModel *model, Error **errp)
return;
}
+ /* PTFF subfunctions might be indicated although kernel support missing */
+ if (!test_bit(S390_FEAT_MULTIPLE_EPOCH, model->features)) {
+ clear_bit(S390_FEAT_PTFF_QSIE, model->features);
+ clear_bit(S390_FEAT_PTFF_QTOUE, model->features);
+ clear_bit(S390_FEAT_PTFF_STOE, model->features);
+ clear_bit(S390_FEAT_PTFF_STOUE, model->features);
+ }
+
/* with cpu model support, CMM is only indicated if really available */
if (kvm_s390_cmma_available()) {
set_bit(S390_FEAT_CMM, model->features);
diff --git a/target/s390x/kvm_s390x.h b/target/s390x/kvm_s390x.h
index 79b35946f3..7a3b862eea 100644
--- a/target/s390x/kvm_s390x.h
+++ b/target/s390x/kvm_s390x.h
@@ -12,17 +12,12 @@
struct kvm_s390_irq;
-void kvm_s390_floating_interrupt(struct kvm_s390_irq *irq);
-void kvm_s390_service_interrupt(uint32_t parm);
+void kvm_s390_floating_interrupt_legacy(struct kvm_s390_irq *irq);
void kvm_s390_vcpu_interrupt(S390CPU *cpu, struct kvm_s390_irq *irq);
void kvm_s390_access_exception(S390CPU *cpu, uint16_t code, uint64_t te_code);
int kvm_s390_mem_op(S390CPU *cpu, vaddr addr, uint8_t ar, void *hostbuf,
int len, bool is_write);
void kvm_s390_program_interrupt(S390CPU *cpu, uint16_t code);
-void kvm_s390_io_interrupt(uint16_t subchannel_id,
- uint16_t subchannel_nr, uint32_t io_int_parm,
- uint32_t io_int_word);
-void kvm_s390_crw_mchk(void);
int kvm_s390_set_cpu_state(S390CPU *cpu, uint8_t cpu_state);
void kvm_s390_vcpu_interrupt_pre_save(S390CPU *cpu);
int kvm_s390_vcpu_interrupt_post_load(S390CPU *cpu);
@@ -44,7 +39,4 @@ void kvm_s390_crypto_reset(void);
void kvm_s390_restart_interrupt(S390CPU *cpu);
void kvm_s390_stop_interrupt(S390CPU *cpu);
-/* implemented outside of target/s390x/ */
-int kvm_s390_inject_flic(struct kvm_s390_irq *irq);
-
#endif /* KVM_S390X_H */
diff --git a/target/s390x/misc_helper.c b/target/s390x/misc_helper.c
index 86da6aab7e..e0b23c1fd1 100644
--- a/target/s390x/misc_helper.c
+++ b/target/s390x/misc_helper.c
@@ -36,6 +36,10 @@
#include "hw/s390x/ebcdic.h"
#include "hw/s390x/s390-virtio-hcall.h"
#include "hw/s390x/sclp.h"
+#include "hw/s390x/s390_flic.h"
+#include "hw/s390x/ioinst.h"
+#include "hw/s390x/s390-pci-inst.h"
+#include "hw/boards.h"
#endif
/* #define DEBUG_HELPER */
@@ -194,132 +198,148 @@ void HELPER(spt)(CPUS390XState *env, uint64_t time)
}
/* Store System Information */
-uint32_t HELPER(stsi)(CPUS390XState *env, uint64_t a0,
- uint64_t r0, uint64_t r1)
+uint32_t HELPER(stsi)(CPUS390XState *env, uint64_t a0, uint64_t r0, uint64_t r1)
{
+ const uintptr_t ra = GETPC();
+ const uint32_t sel1 = r0 & STSI_R0_SEL1_MASK;
+ const uint32_t sel2 = r1 & STSI_R1_SEL2_MASK;
+ const MachineState *ms = MACHINE(qdev_get_machine());
+ uint16_t total_cpus = 0, conf_cpus = 0, reserved_cpus = 0;
S390CPU *cpu = s390_env_get_cpu(env);
- int cc = 0;
- int sel1, sel2;
+ SysIB sysib = { 0 };
+ int i, cc = 0;
+
+ if ((r0 & STSI_R0_FC_MASK) > STSI_R0_FC_LEVEL_3) {
+ /* invalid function code: no other checks are performed */
+ return 3;
+ }
- if ((r0 & STSI_LEVEL_MASK) <= STSI_LEVEL_3 &&
- ((r0 & STSI_R0_RESERVED_MASK) || (r1 & STSI_R1_RESERVED_MASK))) {
- /* valid function code, invalid reserved bits */
- s390_program_interrupt(env, PGM_SPECIFICATION, 4, GETPC());
+ if ((r0 & STSI_R0_RESERVED_MASK) || (r1 & STSI_R1_RESERVED_MASK)) {
+ s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra);
}
- sel1 = r0 & STSI_R0_SEL1_MASK;
- sel2 = r1 & STSI_R1_SEL2_MASK;
+ if ((r0 & STSI_R0_FC_MASK) == STSI_R0_FC_CURRENT) {
+ /* query the current level: no further checks are performed */
+ env->regs[0] = STSI_R0_FC_LEVEL_3;
+ return 0;
+ }
+
+ if (a0 & ~TARGET_PAGE_MASK) {
+ s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra);
+ }
- /* XXX: spec exception if sysib is not 4k-aligned */
+ /* count the cpus and split them into configured and reserved ones */
+ for (i = 0; i < ms->possible_cpus->len; i++) {
+ total_cpus++;
+ if (ms->possible_cpus->cpus[i].cpu) {
+ conf_cpus++;
+ } else {
+ reserved_cpus++;
+ }
+ }
- switch (r0 & STSI_LEVEL_MASK) {
- case STSI_LEVEL_1:
+ /*
+ * In theory, we could report Level 1 / Level 2 as current. However,
+ * the Linux kernel will detect this as running under LPAR and assume
+ * that we have a sclp linemode console (which is always present on
+ * LPAR, but not the default for QEMU), therefore not displaying boot
+ * messages and making booting a Linux kernel under TCG harder.
+ *
+ * For now we fake the same SMP configuration on all levels.
+ *
+ * TODO: We could later make the level configurable via the machine
+ * and change defaults (linemode console) based on machine type
+ * and accelerator.
+ */
+ switch (r0 & STSI_R0_FC_MASK) {
+ case STSI_R0_FC_LEVEL_1:
if ((sel1 == 1) && (sel2 == 1)) {
/* Basic Machine Configuration */
- struct sysib_111 sysib;
char type[5] = {};
- memset(&sysib, 0, sizeof(sysib));
- ebcdic_put(sysib.manuf, "QEMU ", 16);
+ ebcdic_put(sysib.sysib_111.manuf, "QEMU ", 16);
/* same as machine type number in STORE CPU ID, but in EBCDIC */
snprintf(type, ARRAY_SIZE(type), "%X", cpu->model->def->type);
- ebcdic_put(sysib.type, type, 4);
+ ebcdic_put(sysib.sysib_111.type, type, 4);
/* model number (not stored in STORE CPU ID for z/Architecure) */
- ebcdic_put(sysib.model, "QEMU ", 16);
- ebcdic_put(sysib.sequence, "QEMU ", 16);
- ebcdic_put(sysib.plant, "QEMU", 4);
- cpu_physical_memory_write(a0, &sysib, sizeof(sysib));
+ ebcdic_put(sysib.sysib_111.model, "QEMU ", 16);
+ ebcdic_put(sysib.sysib_111.sequence, "QEMU ", 16);
+ ebcdic_put(sysib.sysib_111.plant, "QEMU", 4);
} else if ((sel1 == 2) && (sel2 == 1)) {
/* Basic Machine CPU */
- struct sysib_121 sysib;
-
- memset(&sysib, 0, sizeof(sysib));
- /* XXX make different for different CPUs? */
- ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
- ebcdic_put(sysib.plant, "QEMU", 4);
- stw_p(&sysib.cpu_addr, env->core_id);
- cpu_physical_memory_write(a0, &sysib, sizeof(sysib));
+ ebcdic_put(sysib.sysib_121.sequence, "QEMUQEMUQEMUQEMU", 16);
+ ebcdic_put(sysib.sysib_121.plant, "QEMU", 4);
+ sysib.sysib_121.cpu_addr = cpu_to_be16(env->core_id);
} else if ((sel1 == 2) && (sel2 == 2)) {
/* Basic Machine CPUs */
- struct sysib_122 sysib;
-
- memset(&sysib, 0, sizeof(sysib));
- stl_p(&sysib.capability, 0x443afc29);
- /* XXX change when SMP comes */
- stw_p(&sysib.total_cpus, 1);
- stw_p(&sysib.active_cpus, 1);
- stw_p(&sysib.standby_cpus, 0);
- stw_p(&sysib.reserved_cpus, 0);
- cpu_physical_memory_write(a0, &sysib, sizeof(sysib));
+ sysib.sysib_122.capability = cpu_to_be32(0x443afc29);
+ sysib.sysib_122.total_cpus = cpu_to_be16(total_cpus);
+ sysib.sysib_122.conf_cpus = cpu_to_be16(conf_cpus);
+ sysib.sysib_122.reserved_cpus = cpu_to_be16(reserved_cpus);
} else {
cc = 3;
}
break;
- case STSI_LEVEL_2:
- {
- if ((sel1 == 2) && (sel2 == 1)) {
- /* LPAR CPU */
- struct sysib_221 sysib;
-
- memset(&sysib, 0, sizeof(sysib));
- /* XXX make different for different CPUs? */
- ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
- ebcdic_put(sysib.plant, "QEMU", 4);
- stw_p(&sysib.cpu_addr, env->core_id);
- stw_p(&sysib.cpu_id, 0);
- cpu_physical_memory_write(a0, &sysib, sizeof(sysib));
- } else if ((sel1 == 2) && (sel2 == 2)) {
- /* LPAR CPUs */
- struct sysib_222 sysib;
-
- memset(&sysib, 0, sizeof(sysib));
- stw_p(&sysib.lpar_num, 0);
- sysib.lcpuc = 0;
- /* XXX change when SMP comes */
- stw_p(&sysib.total_cpus, 1);
- stw_p(&sysib.conf_cpus, 1);
- stw_p(&sysib.standby_cpus, 0);
- stw_p(&sysib.reserved_cpus, 0);
- ebcdic_put(sysib.name, "QEMU ", 8);
- stl_p(&sysib.caf, 1000);
- stw_p(&sysib.dedicated_cpus, 0);
- stw_p(&sysib.shared_cpus, 0);
- cpu_physical_memory_write(a0, &sysib, sizeof(sysib));
- } else {
- cc = 3;
- }
- break;
+ case STSI_R0_FC_LEVEL_2:
+ if ((sel1 == 2) && (sel2 == 1)) {
+ /* LPAR CPU */
+ ebcdic_put(sysib.sysib_221.sequence, "QEMUQEMUQEMUQEMU", 16);
+ ebcdic_put(sysib.sysib_221.plant, "QEMU", 4);
+ sysib.sysib_221.cpu_addr = cpu_to_be16(env->core_id);
+ } else if ((sel1 == 2) && (sel2 == 2)) {
+ /* LPAR CPUs */
+ sysib.sysib_222.lcpuc = 0x80; /* dedicated */
+ sysib.sysib_222.total_cpus = cpu_to_be16(total_cpus);
+ sysib.sysib_222.conf_cpus = cpu_to_be16(conf_cpus);
+ sysib.sysib_222.reserved_cpus = cpu_to_be16(reserved_cpus);
+ ebcdic_put(sysib.sysib_222.name, "QEMU ", 8);
+ sysib.sysib_222.caf = cpu_to_be32(1000);
+ sysib.sysib_222.dedicated_cpus = cpu_to_be16(conf_cpus);
+ } else {
+ cc = 3;
}
- case STSI_LEVEL_3:
- {
- if ((sel1 == 2) && (sel2 == 2)) {
- /* VM CPUs */
- struct sysib_322 sysib;
-
- memset(&sysib, 0, sizeof(sysib));
- sysib.count = 1;
- /* XXX change when SMP comes */
- stw_p(&sysib.vm[0].total_cpus, 1);
- stw_p(&sysib.vm[0].conf_cpus, 1);
- stw_p(&sysib.vm[0].standby_cpus, 0);
- stw_p(&sysib.vm[0].reserved_cpus, 0);
- ebcdic_put(sysib.vm[0].name, "KVMguest", 8);
- stl_p(&sysib.vm[0].caf, 1000);
- ebcdic_put(sysib.vm[0].cpi, "KVM/Linux ", 16);
- cpu_physical_memory_write(a0, &sysib, sizeof(sysib));
+ break;
+ case STSI_R0_FC_LEVEL_3:
+ if ((sel1 == 2) && (sel2 == 2)) {
+ /* VM CPUs */
+ sysib.sysib_322.count = 1;
+ sysib.sysib_322.vm[0].total_cpus = cpu_to_be16(total_cpus);
+ sysib.sysib_322.vm[0].conf_cpus = cpu_to_be16(conf_cpus);
+ sysib.sysib_322.vm[0].reserved_cpus = cpu_to_be16(reserved_cpus);
+ sysib.sysib_322.vm[0].caf = cpu_to_be32(1000);
+ /* Linux kernel uses this to distinguish us from z/VM */
+ ebcdic_put(sysib.sysib_322.vm[0].cpi, "KVM/Linux ", 16);
+ sysib.sysib_322.vm[0].ext_name_encoding = 2; /* UTF-8 */
+
+ /* If our VM has a name, use the real name */
+ if (qemu_name) {
+ memset(sysib.sysib_322.vm[0].name, 0x40,
+ sizeof(sysib.sysib_322.vm[0].name));
+ ebcdic_put(sysib.sysib_322.vm[0].name, qemu_name,
+ MIN(sizeof(sysib.sysib_322.vm[0].name),
+ strlen(qemu_name)));
+ strncpy((char *)sysib.sysib_322.ext_names[0], qemu_name,
+ sizeof(sysib.sysib_322.ext_names[0]));
} else {
- cc = 3;
+ ebcdic_put(sysib.sysib_322.vm[0].name, "TCGguest", 8);
+ strcpy((char *)sysib.sysib_322.ext_names[0], "TCGguest");
}
- break;
+
+ /* add the uuid */
+ memcpy(sysib.sysib_322.vm[0].uuid, &qemu_uuid,
+ sizeof(sysib.sysib_322.vm[0].uuid));
+ } else {
+ cc = 3;
}
- case STSI_LEVEL_CURRENT:
- env->regs[0] = STSI_LEVEL_3;
- break;
- default:
- cc = 3;
break;
}
+ if (cc == 0) {
+ if (s390_cpu_virt_mem_write(cpu, a0, 0, &sysib, sizeof(sysib))) {
+ s390_cpu_virt_mem_handle_exc(cpu, ra);
+ }
+ }
+
return cc;
}
@@ -429,6 +449,59 @@ void HELPER(stsch)(CPUS390XState *env, uint64_t r1, uint64_t inst)
qemu_mutex_unlock_iothread();
}
+uint32_t HELPER(tpi)(CPUS390XState *env, uint64_t addr)
+{
+ const uintptr_t ra = GETPC();
+ S390CPU *cpu = s390_env_get_cpu(env);
+ QEMUS390FLICState *flic = s390_get_qemu_flic(s390_get_flic());
+ QEMUS390FlicIO *io = NULL;
+ LowCore *lowcore;
+
+ if (addr & 0x3) {
+ s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra);
+ }
+
+ qemu_mutex_lock_iothread();
+ io = qemu_s390_flic_dequeue_io(flic, env->cregs[6]);
+ if (!io) {
+ qemu_mutex_unlock_iothread();
+ return 0;
+ }
+
+ if (addr) {
+ struct {
+ uint16_t id;
+ uint16_t nr;
+ uint32_t parm;
+ } intc = {
+ .id = cpu_to_be16(io->id),
+ .nr = cpu_to_be16(io->nr),
+ .parm = cpu_to_be32(io->parm),
+ };
+
+ if (s390_cpu_virt_mem_write(cpu, addr, 0, &intc, sizeof(intc))) {
+ /* writing failed, reinject and properly clean up */
+ s390_io_interrupt(io->id, io->nr, io->parm, io->word);
+ qemu_mutex_unlock_iothread();
+ g_free(io);
+ s390_cpu_virt_mem_handle_exc(cpu, ra);
+ return 0;
+ }
+ } else {
+ /* no protection applies */
+ lowcore = cpu_map_lowcore(env);
+ lowcore->subchannel_id = cpu_to_be16(io->id);
+ lowcore->subchannel_nr = cpu_to_be16(io->nr);
+ lowcore->io_int_parm = cpu_to_be32(io->parm);
+ lowcore->io_int_word = cpu_to_be32(io->word);
+ cpu_unmap_lowcore(lowcore);
+ }
+
+ g_free(io);
+ qemu_mutex_unlock_iothread();
+ return 1;
+}
+
void HELPER(tsch)(CPUS390XState *env, uint64_t r1, uint64_t inst)
{
S390CPU *cpu = s390_env_get_cpu(env);
@@ -560,3 +633,91 @@ uint32_t HELPER(stfle)(CPUS390XState *env, uint64_t addr)
env->regs[0] = deposit64(env->regs[0], 0, 8, (max_bytes / 8) - 1);
return count_bytes >= max_bytes ? 0 : 3;
}
+
+#ifndef CONFIG_USER_ONLY
+/*
+ * Note: we ignore any return code of the functions called for the pci
+ * instructions, as the only time they return !0 is when the stub is
+ * called, and in that case we didn't even offer the zpci facility.
+ * The only exception is SIC, where program checks need to be handled
+ * by the caller.
+ */
+void HELPER(clp)(CPUS390XState *env, uint32_t r2)
+{
+ S390CPU *cpu = s390_env_get_cpu(env);
+
+ qemu_mutex_lock_iothread();
+ clp_service_call(cpu, r2, GETPC());
+ qemu_mutex_unlock_iothread();
+}
+
+void HELPER(pcilg)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+{
+ S390CPU *cpu = s390_env_get_cpu(env);
+
+ qemu_mutex_lock_iothread();
+ pcilg_service_call(cpu, r1, r2, GETPC());
+ qemu_mutex_unlock_iothread();
+}
+
+void HELPER(pcistg)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+{
+ S390CPU *cpu = s390_env_get_cpu(env);
+
+ qemu_mutex_lock_iothread();
+ pcistg_service_call(cpu, r1, r2, GETPC());
+ qemu_mutex_unlock_iothread();
+}
+
+void HELPER(stpcifc)(CPUS390XState *env, uint32_t r1, uint64_t fiba,
+ uint32_t ar)
+{
+ S390CPU *cpu = s390_env_get_cpu(env);
+
+ qemu_mutex_lock_iothread();
+ stpcifc_service_call(cpu, r1, fiba, ar, GETPC());
+ qemu_mutex_unlock_iothread();
+}
+
+void HELPER(sic)(CPUS390XState *env, uint64_t r1, uint64_t r3)
+{
+ int r;
+
+ qemu_mutex_lock_iothread();
+ r = css_do_sic(env, (r3 >> 27) & 0x7, r1 & 0xffff);
+ qemu_mutex_unlock_iothread();
+ /* css_do_sic() may actually return a PGM_xxx value to inject */
+ if (r) {
+ s390_program_interrupt(env, -r, 4, GETPC());
+ }
+}
+
+void HELPER(rpcit)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+{
+ S390CPU *cpu = s390_env_get_cpu(env);
+
+ qemu_mutex_lock_iothread();
+ rpcit_service_call(cpu, r1, r2, GETPC());
+ qemu_mutex_unlock_iothread();
+}
+
+void HELPER(pcistb)(CPUS390XState *env, uint32_t r1, uint32_t r3,
+ uint64_t gaddr, uint32_t ar)
+{
+ S390CPU *cpu = s390_env_get_cpu(env);
+
+ qemu_mutex_lock_iothread();
+ pcistb_service_call(cpu, r1, r3, gaddr, ar, GETPC());
+ qemu_mutex_unlock_iothread();
+}
+
+void HELPER(mpcifc)(CPUS390XState *env, uint32_t r1, uint64_t fiba,
+ uint32_t ar)
+{
+ S390CPU *cpu = s390_env_get_cpu(env);
+
+ qemu_mutex_lock_iothread();
+ mpcifc_service_call(cpu, r1, fiba, ar, GETPC());
+ qemu_mutex_unlock_iothread();
+}
+#endif
diff --git a/target/s390x/translate.c b/target/s390x/translate.c
index df0b41606d..b470d691d3 100644
--- a/target/s390x/translate.c
+++ b/target/s390x/translate.c
@@ -4199,6 +4199,14 @@ static ExitStatus op_stcrw(DisasContext *s, DisasOps *o)
return NO_EXIT;
}
+static ExitStatus op_tpi(DisasContext *s, DisasOps *o)
+{
+ check_privileged(s);
+ gen_helper_tpi(cc_op, cpu_env, o->addr1);
+ set_cc_static(s);
+ return NO_EXIT;
+}
+
static ExitStatus op_tsch(DisasContext *s, DisasOps *o)
{
check_privileged(s);
@@ -4777,6 +4785,106 @@ static ExitStatus op_zero2(DisasContext *s, DisasOps *o)
return NO_EXIT;
}
+#ifndef CONFIG_USER_ONLY
+static ExitStatus op_clp(DisasContext *s, DisasOps *o)
+{
+ TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2));
+
+ check_privileged(s);
+ gen_helper_clp(cpu_env, r2);
+ tcg_temp_free_i32(r2);
+ set_cc_static(s);
+ return NO_EXIT;
+}
+
+static ExitStatus op_pcilg(DisasContext *s, DisasOps *o)
+{
+ TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+ TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2));
+
+ check_privileged(s);
+ gen_helper_pcilg(cpu_env, r1, r2);
+ tcg_temp_free_i32(r1);
+ tcg_temp_free_i32(r2);
+ set_cc_static(s);
+ return NO_EXIT;
+}
+
+static ExitStatus op_pcistg(DisasContext *s, DisasOps *o)
+{
+ TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+ TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2));
+
+ check_privileged(s);
+ gen_helper_pcistg(cpu_env, r1, r2);
+ tcg_temp_free_i32(r1);
+ tcg_temp_free_i32(r2);
+ set_cc_static(s);
+ return NO_EXIT;
+}
+
+static ExitStatus op_stpcifc(DisasContext *s, DisasOps *o)
+{
+ TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+ TCGv_i32 ar = tcg_const_i32(get_field(s->fields, b2));
+
+ check_privileged(s);
+ gen_helper_stpcifc(cpu_env, r1, o->addr1, ar);
+ tcg_temp_free_i32(ar);
+ tcg_temp_free_i32(r1);
+ set_cc_static(s);
+ return NO_EXIT;
+}
+
+static ExitStatus op_sic(DisasContext *s, DisasOps *o)
+{
+ check_privileged(s);
+ gen_helper_sic(cpu_env, o->in1, o->in2);
+ return NO_EXIT;
+}
+
+static ExitStatus op_rpcit(DisasContext *s, DisasOps *o)
+{
+ TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+ TCGv_i32 r2 = tcg_const_i32(get_field(s->fields, r2));
+
+ check_privileged(s);
+ gen_helper_rpcit(cpu_env, r1, r2);
+ tcg_temp_free_i32(r1);
+ tcg_temp_free_i32(r2);
+ set_cc_static(s);
+ return NO_EXIT;
+}
+
+static ExitStatus op_pcistb(DisasContext *s, DisasOps *o)
+{
+ TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+ TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
+ TCGv_i32 ar = tcg_const_i32(get_field(s->fields, b2));
+
+ check_privileged(s);
+ gen_helper_pcistb(cpu_env, r1, r3, o->addr1, ar);
+ tcg_temp_free_i32(ar);
+ tcg_temp_free_i32(r1);
+ tcg_temp_free_i32(r3);
+ set_cc_static(s);
+ return NO_EXIT;
+}
+
+static ExitStatus op_mpcifc(DisasContext *s, DisasOps *o)
+{
+ TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
+ TCGv_i32 ar = tcg_const_i32(get_field(s->fields, b2));
+
+ check_privileged(s);
+ gen_helper_mpcifc(cpu_env, r1, o->addr1, ar);
+ tcg_temp_free_i32(ar);
+ tcg_temp_free_i32(r1);
+ set_cc_static(s);
+ return NO_EXIT;
+}
+#endif
+
/* ====================================================================== */
/* The "Cc OUTput" generators. Given the generated output (and in some cases
the original inputs), update the various cc data structures in order to
@@ -5708,6 +5816,8 @@ enum DisasInsnEnum {
#define FAC_MSA4 S390_FEAT_MSA_EXT_4 /* msa-extension-4 facility */
#define FAC_MSA5 S390_FEAT_MSA_EXT_5 /* msa-extension-5 facility */
#define FAC_ECT S390_FEAT_EXTRACT_CPU_TIME
+#define FAC_PCI S390_FEAT_ZPCI /* z/PCI facility */
+#define FAC_AIS S390_FEAT_ADAPTER_INT_SUPPRESSION
static const DisasInsn insn_info[] = {
#include "insn-data.def"
generated by cgit v1.2.3 (git 2.26.2) at 2025-01-09 13:37:42 +0000