target-s390: Perform COMPARE AND SWAP inline

Still no proper solution for CONFIG_USER_ONLY, but the system
version is significantly better.

Signed-off-by: Richard Henderson <rth@twiddle.net>

4 files changed, 124 insertions, 96 deletions

diff --git a/target-s390x/helper.h b/target-s390x/helper.h
index b7376339cd..a5844b3ee0 100644
--- a/target-s390x/helper.h
+++ b/target-s390x/helper.h
@@ -17,9 +17,6 @@ DEF_HELPER_4(srst, i64, env, i64, i64, i64)
 DEF_HELPER_4(clst, i64, env, i64, i64, i64)
 DEF_HELPER_4(mvpg, void, env, i64, i64, i64)
 DEF_HELPER_4(mvst, i64, env, i64, i64, i64)
-DEF_HELPER_4(csg, i64, env, i64, i64, i64)
-DEF_HELPER_4(cdsg, i32, env, i32, i64, i32)
-DEF_HELPER_4(cs, i64, env, i64, i64, i64)
 DEF_HELPER_5(ex, i32, env, i32, i64, i64, i64)
 DEF_HELPER_FLAGS_1(abs_i32, TCG_CALL_NO_RWG_SE, i32, s32)
 DEF_HELPER_FLAGS_1(nabs_i32, TCG_CALL_NO_RWG_SE, s32, s32)
diff --git a/target-s390x/insn-data.def b/target-s390x/insn-data.def
index 480f1fdf51..b42ebb6a1a 100644
--- a/target-s390x/insn-data.def
+++ b/target-s390x/insn-data.def
@@ -193,13 +193,13 @@
     D(0xec7d, CLGIJ,   RIE_c, GIE, r1_o, i2_8u, 0, 0, cj, 0, 1)
 
 /* COMPARE AND SWAP */
-    C(0xba00, CS,      RS_a,  Z,   r1_o, a2, new, r1_32, cs, 0)
-    C(0xeb14, CSY,     RSY_a, LD,  r1_o, a2, new, r1_32, cs, 0)
-    C(0xeb30, CSG,     RSY_a, Z,   r1_o, a2, r1, 0, csg, 0)
+    D(0xba00, CS,      RS_a,  Z,   r3_32u, r1_32u, new, r1_32, cs, 0, 0)
+    D(0xeb14, CSY,     RSY_a, LD,  r3_32u, r1_32u, new, r1_32, cs, 0, 0)
+    D(0xeb30, CSG,     RSY_a, Z,   r3_o, r1_o, new, r1, cs, 0, 1)
 /* COMPARE DOUBLE AND SWAP */
-    C(0xbb00, CDS,     RS_a,  Z,   r1_D32, a2, new, r1_D32, cds, 0)
-    C(0xeb31, CDSY,    RSY_a, LD,  r1_D32, a2, new, r1_D32, cds, 0)
-    C(0xeb3e, CDSG,    RSY_a, Z,   0, a2, 0, 0, cdsg, 0)
+    D(0xbb00, CDS,     RS_a,  Z,   r3_D32, r1_D32, new, r1_D32, cs, 0, 1)
+    D(0xeb31, CDSY,    RSY_a, LD,  r3_D32, r1_D32, new, r1_D32, cs, 0, 1)
+    C(0xeb3e, CDSG,    RSY_a, Z,   0, 0, 0, 0, cdsg, 0)
 
 /* COMPARE AND TRAP */
     D(0xb972, CRT,     RRF_c, GIE, r1_32s, r2_32s, 0, 0, ct, 0, 0)
diff --git a/target-s390x/mem_helper.c b/target-s390x/mem_helper.c
index 0e9bbd4035..372334b3c8 100644
--- a/target-s390x/mem_helper.c
+++ b/target-s390x/mem_helper.c
@@ -442,59 +442,6 @@ uint64_t HELPER(mvst)(CPUS390XState *env, uint64_t c, uint64_t d, uint64_t s)
     return d + len;
 }
 
-/* compare and swap 64-bit */
-uint64_t HELPER(csg)(CPUS390XState *env, uint64_t r1, uint64_t a2, uint64_t r3)
-{
-    /* FIXME: locking? */
-    uint64_t v2 = cpu_ldq_data(env, a2);
-    if (r1 == v2) {
-        cpu_stq_data(env, a2, r3);
-        env->cc_op = 0;
-        return r1;
-    } else {
-        env->cc_op = 1;
-        return v2;
-    }
-}
-
-/* compare double and swap 64-bit */
-uint32_t HELPER(cdsg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    uint64_t v2_hi = cpu_ldq_data(env, a2);
-    uint64_t v2_lo = cpu_ldq_data(env, a2 + 8);
-    uint64_t v1_hi = env->regs[r1];
-    uint64_t v1_lo = env->regs[r1 + 1];
-
-    if ((v1_hi == v2_hi) && (v1_lo == v2_lo)) {
-        cc = 0;
-        cpu_stq_data(env, a2, env->regs[r3]);
-        cpu_stq_data(env, a2 + 8, env->regs[r3 + 1]);
-    } else {
-        cc = 1;
-        env->regs[r1] = v2_hi;
-        env->regs[r1 + 1] = v2_lo;
-    }
-
-    return cc;
-}
-
-/* compare and swap 32-bit */
-uint64_t HELPER(cs)(CPUS390XState *env, uint64_t r1, uint64_t a2, uint64_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t v2 = cpu_ldl_data(env, a2);
-    if ((uint32_t)r1 == v2) {
-        cpu_stl_data(env, a2, (uint32_t)r3);
-        env->cc_op = 0;
-        return r1;
-    } else {
-        env->cc_op = 1;
-        return v2;
-    }
-}
-
 static uint32_t helper_icm(CPUS390XState *env, uint32_t r1, uint64_t address,
                            uint32_t mask)
 {
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index 0ad78d2eb4..a57296c64f 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -1078,8 +1078,9 @@ typedef struct {
 
 #define SPEC_r1_even    1
 #define SPEC_r2_even    2
-#define SPEC_r1_f128    4
-#define SPEC_r2_f128    8
+#define SPEC_r3_even    4
+#define SPEC_r1_f128    8
+#define SPEC_r2_f128    16
 
 /* Return values from translate_one, indicating the state of the TB.  */
 typedef enum {
@@ -1124,9 +1125,9 @@ typedef enum DisasFacility {
 
 struct DisasInsn {
     unsigned opc:16;
-    DisasFormat fmt:6;
-    DisasFacility fac:6;
-    unsigned spec:4;
+    DisasFormat fmt:8;
+    DisasFacility fac:8;
+    unsigned spec:8;
 
     const char *name;
 
@@ -1828,18 +1829,102 @@ static ExitStatus op_cps(DisasContext *s, DisasOps *o)
 
 static ExitStatus op_cs(DisasContext *s, DisasOps *o)
 {
-    int r3 = get_field(s->fields, r3);
-    potential_page_fault(s);
-    gen_helper_cs(o->out, cpu_env, o->in1, o->in2, regs[r3]);
+    /* FIXME: needs an atomic solution for CONFIG_USER_ONLY.  */
+    int d2 = get_field(s->fields, d2);
+    int b2 = get_field(s->fields, b2);
+    int is_64 = s->insn->data;
+    TCGv_i64 addr, mem, cc, z;
+
+    /* Note that in1 = R3 (new value) and
+       in2 = (zero-extended) R1 (expected value).  */
+
+    /* Load the memory into the (temporary) output.  While the PoO only talks
+       about moving the memory to R1 on inequality, if we include equality it
+       means that R1 is equal to the memory in all conditions.  */
+    addr = get_address(s, 0, b2, d2);
+    if (is_64) {
+        tcg_gen_qemu_ld64(o->out, addr, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_ld32u(o->out, addr, get_mem_index(s));
+    }
+
+    /* Are the memory and expected values (un)equal?  Note that this setcond
+       produces the output CC value, thus the NE sense of the test.  */
+    cc = tcg_temp_new_i64();
+    tcg_gen_setcond_i64(TCG_COND_NE, cc, o->in2, o->out);
+
+    /* If the memory and expected values are equal (CC==0), copy R3 to MEM.
+       Recall that we are allowed to unconditionally issue the store (and
+       thus any possible write trap), so (re-)store the original contents
+       of MEM in case of inequality.  */
+    z = tcg_const_i64(0);
+    mem = tcg_temp_new_i64();
+    tcg_gen_movcond_i64(TCG_COND_EQ, mem, cc, z, o->in1, o->out);
+    if (is_64) {
+        tcg_gen_qemu_st64(mem, addr, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_st32(mem, addr, get_mem_index(s));
+    }
+    tcg_temp_free_i64(z);
+    tcg_temp_free_i64(mem);
+    tcg_temp_free_i64(addr);
+
+    /* Store CC back to cc_op.  Wait until after the store so that any
+       exception gets the old cc_op value.  */
+    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_temp_free_i64(cc);
     set_cc_static(s);
     return NO_EXIT;
 }
 
-static ExitStatus op_csg(DisasContext *s, DisasOps *o)
+static ExitStatus op_cdsg(DisasContext *s, DisasOps *o)
 {
+    /* FIXME: needs an atomic solution for CONFIG_USER_ONLY.  */
+    int r1 = get_field(s->fields, r1);
     int r3 = get_field(s->fields, r3);
-    potential_page_fault(s);
-    gen_helper_csg(o->out, cpu_env, o->in1, o->in2, regs[r3]);
+    int d2 = get_field(s->fields, d2);
+    int b2 = get_field(s->fields, b2);
+    TCGv_i64 addrh, addrl, memh, meml, outh, outl, cc, z;
+
+    /* Note that R1:R1+1 = expected value and R3:R3+1 = new value.  */
+
+    addrh = get_address(s, 0, b2, d2);
+    addrl = get_address(s, 0, b2, d2 + 8);
+    outh = tcg_temp_new_i64();
+    outl = tcg_temp_new_i64();
+
+    tcg_gen_qemu_ld64(outh, addrh, get_mem_index(s));
+    tcg_gen_qemu_ld64(outl, addrl, get_mem_index(s));
+
+    /* Fold the double-word compare with arithmetic.  */
+    cc = tcg_temp_new_i64();
+    z = tcg_temp_new_i64();
+    tcg_gen_xor_i64(cc, outh, regs[r1]);
+    tcg_gen_xor_i64(z, outl, regs[r1 + 1]);
+    tcg_gen_or_i64(cc, cc, z);
+    tcg_gen_movi_i64(z, 0);
+    tcg_gen_setcond_i64(TCG_COND_NE, cc, cc, z);
+
+    memh = tcg_temp_new_i64();
+    meml = tcg_temp_new_i64();
+    tcg_gen_movcond_i64(TCG_COND_EQ, memh, cc, z, regs[r3], outh);
+    tcg_gen_movcond_i64(TCG_COND_EQ, meml, cc, z, regs[r3 + 1], outl);
+    tcg_temp_free_i64(z);
+
+    tcg_gen_qemu_st64(memh, addrh, get_mem_index(s));
+    tcg_gen_qemu_st64(meml, addrl, get_mem_index(s));
+    tcg_temp_free_i64(memh);
+    tcg_temp_free_i64(meml);
+    tcg_temp_free_i64(addrh);
+    tcg_temp_free_i64(addrl);
+
+    /* Save back state now that we've passed all exceptions.  */
+    tcg_gen_mov_i64(regs[r1], outh);
+    tcg_gen_mov_i64(regs[r1 + 1], outl);
+    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_temp_free_i64(outh);
+    tcg_temp_free_i64(outl);
+    tcg_temp_free_i64(cc);
     set_cc_static(s);
     return NO_EXIT;
 }
@@ -1856,29 +1941,6 @@ static ExitStatus op_csp(DisasContext *s, DisasOps *o)
 }
 #endif
 
-static ExitStatus op_cds(DisasContext *s, DisasOps *o)
-{
-    int r3 = get_field(s->fields, r3);
-    TCGv_i64 in3 = tcg_temp_new_i64();
-    tcg_gen_deposit_i64(in3, regs[r3 + 1], regs[r3], 32, 32);
-    potential_page_fault(s);
-    gen_helper_csg(o->out, cpu_env, o->in1, o->in2, in3);
-    tcg_temp_free_i64(in3);
-    set_cc_static(s);
-    return NO_EXIT;
-}
-
-static ExitStatus op_cdsg(DisasContext *s, DisasOps *o)
-{
-    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
-    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
-    potential_page_fault(s);
-    /* XXX rewrite in tcg */
-    gen_helper_cdsg(cc_op, cpu_env, r1, o->in2, r3);
-    set_cc_static(s);
-    return NO_EXIT;
-}
-
 static ExitStatus op_cvd(DisasContext *s, DisasOps *o)
 {
     TCGv_i64 t1 = tcg_temp_new_i64();
@@ -4007,6 +4069,14 @@ static void in1_r3_32u(DisasContext *s, DisasFields *f, DisasOps *o)
 }
 #define SPEC_in1_r3_32u 0
 
+static void in1_r3_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r3 = get_field(f, r3);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in1, regs[r3 + 1], regs[r3]);
+}
+#define SPEC_in1_r3_D32 SPEC_r3_even
+
 static void in1_e1(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     o->in1 = load_freg32_i64(get_field(f, r1));
@@ -4121,6 +4191,14 @@ static void in2_r1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
 }
 #define SPEC_in2_r1_32u 0
 
+static void in2_r1_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in2, regs[r1 + 1], regs[r1]);
+}
+#define SPEC_in2_r1_D32 SPEC_r1_even
+
 static void in2_r2(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     o->in2 = load_reg(get_field(f, r2));
@@ -4580,6 +4658,12 @@ static ExitStatus translate_one(CPUS390XState *env, DisasContext *s)
                 excp = PGM_SPECIFICATION;
             }
         }
+        if (spec & SPEC_r3_even) {
+            r = get_field(&f, r3);
+            if (r & 1) {
+                excp = PGM_SPECIFICATION;
+            }
+        }
         if (spec & SPEC_r1_f128) {
             r = get_field(&f, r1);
             if (r > 13) {