target-s390: Perform COMPARE AND SWAP inline

Still no proper solution for CONFIG_USER_ONLY, but the system
version is significantly better.

Signed-off-by: Richard Henderson <rth@twiddle.net>

1 files changed, 118 insertions, 34 deletions

diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index 0ad78d2eb4..a57296c64f 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -1078,8 +1078,9 @@ typedef struct {
 
 #define SPEC_r1_even    1
 #define SPEC_r2_even    2
-#define SPEC_r1_f128    4
-#define SPEC_r2_f128    8
+#define SPEC_r3_even    4
+#define SPEC_r1_f128    8
+#define SPEC_r2_f128    16
 
 /* Return values from translate_one, indicating the state of the TB.  */
 typedef enum {
@@ -1124,9 +1125,9 @@ typedef enum DisasFacility {
 
 struct DisasInsn {
     unsigned opc:16;
-    DisasFormat fmt:6;
-    DisasFacility fac:6;
-    unsigned spec:4;
+    DisasFormat fmt:8;
+    DisasFacility fac:8;
+    unsigned spec:8;
 
     const char *name;
 
@@ -1828,18 +1829,102 @@ static ExitStatus op_cps(DisasContext *s, DisasOps *o)
 
 static ExitStatus op_cs(DisasContext *s, DisasOps *o)
 {
-    int r3 = get_field(s->fields, r3);
-    potential_page_fault(s);
-    gen_helper_cs(o->out, cpu_env, o->in1, o->in2, regs[r3]);
+    /* FIXME: needs an atomic solution for CONFIG_USER_ONLY.  */
+    int d2 = get_field(s->fields, d2);
+    int b2 = get_field(s->fields, b2);
+    int is_64 = s->insn->data;
+    TCGv_i64 addr, mem, cc, z;
+
+    /* Note that in1 = R3 (new value) and
+       in2 = (zero-extended) R1 (expected value).  */
+
+    /* Load the memory into the (temporary) output.  While the PoO only talks
+       about moving the memory to R1 on inequality, if we include equality it
+       means that R1 is equal to the memory in all conditions.  */
+    addr = get_address(s, 0, b2, d2);
+    if (is_64) {
+        tcg_gen_qemu_ld64(o->out, addr, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_ld32u(o->out, addr, get_mem_index(s));
+    }
+
+    /* Are the memory and expected values (un)equal?  Note that this setcond
+       produces the output CC value, thus the NE sense of the test.  */
+    cc = tcg_temp_new_i64();
+    tcg_gen_setcond_i64(TCG_COND_NE, cc, o->in2, o->out);
+
+    /* If the memory and expected values are equal (CC==0), copy R3 to MEM.
+       Recall that we are allowed to unconditionally issue the store (and
+       thus any possible write trap), so (re-)store the original contents
+       of MEM in case of inequality.  */
+    z = tcg_const_i64(0);
+    mem = tcg_temp_new_i64();
+    tcg_gen_movcond_i64(TCG_COND_EQ, mem, cc, z, o->in1, o->out);
+    if (is_64) {
+        tcg_gen_qemu_st64(mem, addr, get_mem_index(s));
+    } else {
+        tcg_gen_qemu_st32(mem, addr, get_mem_index(s));
+    }
+    tcg_temp_free_i64(z);
+    tcg_temp_free_i64(mem);
+    tcg_temp_free_i64(addr);
+
+    /* Store CC back to cc_op.  Wait until after the store so that any
+       exception gets the old cc_op value.  */
+    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_temp_free_i64(cc);
     set_cc_static(s);
     return NO_EXIT;
 }
 
-static ExitStatus op_csg(DisasContext *s, DisasOps *o)
+static ExitStatus op_cdsg(DisasContext *s, DisasOps *o)
 {
+    /* FIXME: needs an atomic solution for CONFIG_USER_ONLY.  */
+    int r1 = get_field(s->fields, r1);
     int r3 = get_field(s->fields, r3);
-    potential_page_fault(s);
-    gen_helper_csg(o->out, cpu_env, o->in1, o->in2, regs[r3]);
+    int d2 = get_field(s->fields, d2);
+    int b2 = get_field(s->fields, b2);
+    TCGv_i64 addrh, addrl, memh, meml, outh, outl, cc, z;
+
+    /* Note that R1:R1+1 = expected value and R3:R3+1 = new value.  */
+
+    addrh = get_address(s, 0, b2, d2);
+    addrl = get_address(s, 0, b2, d2 + 8);
+    outh = tcg_temp_new_i64();
+    outl = tcg_temp_new_i64();
+
+    tcg_gen_qemu_ld64(outh, addrh, get_mem_index(s));
+    tcg_gen_qemu_ld64(outl, addrl, get_mem_index(s));
+
+    /* Fold the double-word compare with arithmetic.  */
+    cc = tcg_temp_new_i64();
+    z = tcg_temp_new_i64();
+    tcg_gen_xor_i64(cc, outh, regs[r1]);
+    tcg_gen_xor_i64(z, outl, regs[r1 + 1]);
+    tcg_gen_or_i64(cc, cc, z);
+    tcg_gen_movi_i64(z, 0);
+    tcg_gen_setcond_i64(TCG_COND_NE, cc, cc, z);
+
+    memh = tcg_temp_new_i64();
+    meml = tcg_temp_new_i64();
+    tcg_gen_movcond_i64(TCG_COND_EQ, memh, cc, z, regs[r3], outh);
+    tcg_gen_movcond_i64(TCG_COND_EQ, meml, cc, z, regs[r3 + 1], outl);
+    tcg_temp_free_i64(z);
+
+    tcg_gen_qemu_st64(memh, addrh, get_mem_index(s));
+    tcg_gen_qemu_st64(meml, addrl, get_mem_index(s));
+    tcg_temp_free_i64(memh);
+    tcg_temp_free_i64(meml);
+    tcg_temp_free_i64(addrh);
+    tcg_temp_free_i64(addrl);
+
+    /* Save back state now that we've passed all exceptions.  */
+    tcg_gen_mov_i64(regs[r1], outh);
+    tcg_gen_mov_i64(regs[r1 + 1], outl);
+    tcg_gen_trunc_i64_i32(cc_op, cc);
+    tcg_temp_free_i64(outh);
+    tcg_temp_free_i64(outl);
+    tcg_temp_free_i64(cc);
     set_cc_static(s);
     return NO_EXIT;
 }
@@ -1856,29 +1941,6 @@ static ExitStatus op_csp(DisasContext *s, DisasOps *o)
 }
 #endif
 
-static ExitStatus op_cds(DisasContext *s, DisasOps *o)
-{
-    int r3 = get_field(s->fields, r3);
-    TCGv_i64 in3 = tcg_temp_new_i64();
-    tcg_gen_deposit_i64(in3, regs[r3 + 1], regs[r3], 32, 32);
-    potential_page_fault(s);
-    gen_helper_csg(o->out, cpu_env, o->in1, o->in2, in3);
-    tcg_temp_free_i64(in3);
-    set_cc_static(s);
-    return NO_EXIT;
-}
-
-static ExitStatus op_cdsg(DisasContext *s, DisasOps *o)
-{
-    TCGv_i32 r1 = tcg_const_i32(get_field(s->fields, r1));
-    TCGv_i32 r3 = tcg_const_i32(get_field(s->fields, r3));
-    potential_page_fault(s);
-    /* XXX rewrite in tcg */
-    gen_helper_cdsg(cc_op, cpu_env, r1, o->in2, r3);
-    set_cc_static(s);
-    return NO_EXIT;
-}
-
 static ExitStatus op_cvd(DisasContext *s, DisasOps *o)
 {
     TCGv_i64 t1 = tcg_temp_new_i64();
@@ -4007,6 +4069,14 @@ static void in1_r3_32u(DisasContext *s, DisasFields *f, DisasOps *o)
 }
 #define SPEC_in1_r3_32u 0
 
+static void in1_r3_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r3 = get_field(f, r3);
+    o->in1 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in1, regs[r3 + 1], regs[r3]);
+}
+#define SPEC_in1_r3_D32 SPEC_r3_even
+
 static void in1_e1(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     o->in1 = load_freg32_i64(get_field(f, r1));
@@ -4121,6 +4191,14 @@ static void in2_r1_32u(DisasContext *s, DisasFields *f, DisasOps *o)
 }
 #define SPEC_in2_r1_32u 0
 
+static void in2_r1_D32(DisasContext *s, DisasFields *f, DisasOps *o)
+{
+    int r1 = get_field(f, r1);
+    o->in2 = tcg_temp_new_i64();
+    tcg_gen_concat32_i64(o->in2, regs[r1 + 1], regs[r1]);
+}
+#define SPEC_in2_r1_D32 SPEC_r1_even
+
 static void in2_r2(DisasContext *s, DisasFields *f, DisasOps *o)
 {
     o->in2 = load_reg(get_field(f, r2));
@@ -4580,6 +4658,12 @@ static ExitStatus translate_one(CPUS390XState *env, DisasContext *s)
                 excp = PGM_SPECIFICATION;
             }
         }
+        if (spec & SPEC_r3_even) {
+            r = get_field(&f, r3);
+            if (r & 1) {
+                excp = PGM_SPECIFICATION;
+            }
+        }
         if (spec & SPEC_r1_f128) {
             r = get_field(&f, r1);
             if (r > 13) {