From 409b9e39bec1b43de982c4708832bac13d4a20ab Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 10 Sep 2024 13:59:36 -0700 Subject: tcg: Return TCGOp from tcg_gen_op[1-6] MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TCGOp to be propagated further in the next patch. Reviewed-by: Alex Bennée Reviewed-by: Pierrick Bouvier Signed-off-by: Richard Henderson --- tcg/tcg-internal.h | 12 ++++++------ tcg/tcg-op.c | 23 +++++++++++++++-------- 2 files changed, 21 insertions(+), 14 deletions(-) (limited to 'tcg') diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index 9b0d982f65..d18f49f5d3 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -92,12 +92,12 @@ TCGTemp *tcg_temp_new_internal(TCGType type, TCGTempKind kind); */ TCGTemp *tcg_constant_internal(TCGType type, int64_t val); -void tcg_gen_op1(TCGOpcode, TCGArg); -void tcg_gen_op2(TCGOpcode, TCGArg, TCGArg); -void tcg_gen_op3(TCGOpcode, TCGArg, TCGArg, TCGArg); -void tcg_gen_op4(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg); -void tcg_gen_op5(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); -void tcg_gen_op6(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); +TCGOp *tcg_gen_op1(TCGOpcode, TCGArg); +TCGOp *tcg_gen_op2(TCGOpcode, TCGArg, TCGArg); +TCGOp *tcg_gen_op3(TCGOpcode, TCGArg, TCGArg, TCGArg); +TCGOp *tcg_gen_op4(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg); +TCGOp *tcg_gen_op5(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); +TCGOp *tcg_gen_op6(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); void vec_gen_2(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg); void vec_gen_3(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg, TCGArg); diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index eff3728622..28c41b37a4 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -37,38 +37,43 @@ */ #define NI __attribute__((noinline)) -void NI tcg_gen_op1(TCGOpcode opc, TCGArg a1) +TCGOp * NI tcg_gen_op1(TCGOpcode opc, TCGArg a1) { TCGOp *op = tcg_emit_op(opc, 1); op->args[0] = a1; + return op; } -void NI tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) +TCGOp * NI tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) { TCGOp *op = tcg_emit_op(opc, 2); op->args[0] = a1; op->args[1] = a2; + return op; } -void NI tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) +TCGOp * NI tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) { TCGOp *op = tcg_emit_op(opc, 3); op->args[0] = a1; op->args[1] = a2; op->args[2] = a3; + return op; } -void NI tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) +TCGOp * NI tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, + TCGArg a3, TCGArg a4) { TCGOp *op = tcg_emit_op(opc, 4); op->args[0] = a1; op->args[1] = a2; op->args[2] = a3; op->args[3] = a4; + return op; } -void NI tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, - TCGArg a4, TCGArg a5) +TCGOp * NI tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, + TCGArg a3, TCGArg a4, TCGArg a5) { TCGOp *op = tcg_emit_op(opc, 5); op->args[0] = a1; @@ -76,10 +81,11 @@ void NI tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, op->args[2] = a3; op->args[3] = a4; op->args[4] = a5; + return op; } -void NI tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, - TCGArg a4, TCGArg a5, TCGArg a6) +TCGOp * NI tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, + TCGArg a4, TCGArg a5, TCGArg a6) { TCGOp *op = tcg_emit_op(opc, 6); op->args[0] = a1; @@ -88,6 +94,7 @@ void NI tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, op->args[3] = a4; op->args[4] = a5; op->args[5] = a6; + return op; } /* -- cgit v1.2.3 From 83ac625c2bb14cc2991adedc9ac4f3f9ab3b096d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 10 Sep 2024 14:17:30 -0700 Subject: tcg: Propagate new TCGOp to add_as_label_use The use of tcg_last_op does not interact well with TCGContext.emit_before_op, resulting in the label being linked to something other than the branch op. In this case it is easier to simply collect the emitted branch op and pass it directly to add_as_label_use. Reported-by: Elisha Hollander Reviewed-by: Pierrick Bouvier Signed-off-by: Richard Henderson --- tcg/tcg-op.c | 63 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 32 insertions(+), 31 deletions(-) (limited to 'tcg') diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index 28c41b37a4..4a7e705367 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -117,9 +117,9 @@ static void DNI tcg_gen_op1_i64(TCGOpcode opc, TCGv_i64 a1) tcg_gen_op1(opc, tcgv_i64_arg(a1)); } -static void DNI tcg_gen_op1i(TCGOpcode opc, TCGArg a1) +static TCGOp * DNI tcg_gen_op1i(TCGOpcode opc, TCGArg a1) { - tcg_gen_op1(opc, a1); + return tcg_gen_op1(opc, a1); } static void DNI tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2) @@ -196,16 +196,16 @@ static void DNI tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, tcgv_i64_arg(a3), a4); } -static void DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGArg a3, TCGArg a4) +static TCGOp * DNI tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGArg a3, TCGArg a4) { - tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); + return tcg_gen_op4(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), a3, a4); } -static void DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, - TCGArg a3, TCGArg a4) +static TCGOp * DNI tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, + TCGArg a3, TCGArg a4) { - tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); + return tcg_gen_op4(opc, tcgv_i64_arg(a1), tcgv_i64_arg(a2), a3, a4); } static void DNI tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, @@ -270,12 +270,12 @@ static void DNI tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 a1, TCGv_i64 a2, tcgv_i64_arg(a3), tcgv_i64_arg(a4), tcgv_i64_arg(a5), a6); } -static void DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, - TCGv_i32 a3, TCGv_i32 a4, - TCGArg a5, TCGArg a6) +static TCGOp * DNI tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 a1, TCGv_i32 a2, + TCGv_i32 a3, TCGv_i32 a4, + TCGArg a5, TCGArg a6) { - tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), - tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); + return tcg_gen_op6(opc, tcgv_i32_arg(a1), tcgv_i32_arg(a2), + tcgv_i32_arg(a3), tcgv_i32_arg(a4), a5, a6); } /* Generic ops. */ @@ -286,18 +286,17 @@ void gen_set_label(TCGLabel *l) tcg_gen_op1(INDEX_op_set_label, label_arg(l)); } -static void add_last_as_label_use(TCGLabel *l) +static void add_as_label_use(TCGLabel *l, TCGOp *op) { TCGLabelUse *u = tcg_malloc(sizeof(TCGLabelUse)); - u->op = tcg_last_op(); + u->op = op; QSIMPLEQ_INSERT_TAIL(&l->branches, u, next); } void tcg_gen_br(TCGLabel *l) { - tcg_gen_op1(INDEX_op_br, label_arg(l)); - add_last_as_label_use(l); + add_as_label_use(l, tcg_gen_op1(INDEX_op_br, label_arg(l))); } void tcg_gen_mb(TCGBar mb_type) @@ -514,8 +513,9 @@ void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l) if (cond == TCG_COND_ALWAYS) { tcg_gen_br(l); } else if (cond != TCG_COND_NEVER) { - tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l)); - add_last_as_label_use(l); + TCGOp *op = tcg_gen_op4ii_i32(INDEX_op_brcond_i32, + arg1, arg2, cond, label_arg(l)); + add_as_label_use(l, op); } } @@ -1934,15 +1934,16 @@ void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l) if (cond == TCG_COND_ALWAYS) { tcg_gen_br(l); } else if (cond != TCG_COND_NEVER) { + TCGOp *op; if (TCG_TARGET_REG_BITS == 32) { - tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), - TCGV_HIGH(arg1), TCGV_LOW(arg2), - TCGV_HIGH(arg2), cond, label_arg(l)); + op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1), + TCGV_HIGH(arg1), TCGV_LOW(arg2), + TCGV_HIGH(arg2), cond, label_arg(l)); } else { - tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, - label_arg(l)); + op = tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, + label_arg(l)); } - add_last_as_label_use(l); + add_as_label_use(l, op); } } @@ -1953,12 +1954,12 @@ void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l) } else if (cond == TCG_COND_ALWAYS) { tcg_gen_br(l); } else if (cond != TCG_COND_NEVER) { - tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, - TCGV_LOW(arg1), TCGV_HIGH(arg1), - tcg_constant_i32(arg2), - tcg_constant_i32(arg2 >> 32), - cond, label_arg(l)); - add_last_as_label_use(l); + TCGOp *op = tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, + TCGV_LOW(arg1), TCGV_HIGH(arg1), + tcg_constant_i32(arg2), + tcg_constant_i32(arg2 >> 32), + cond, label_arg(l)); + add_as_label_use(l, op); } } -- cgit v1.2.3 From 9d8d5a5b9078a16b4c0862fe54248c5cc8435648 Mon Sep 17 00:00:00 2001 From: TANG Tiancheng Date: Wed, 4 Sep 2024 22:27:26 +0800 Subject: tcg: Fix iteration step in 32-bit gvec operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The loop in the 32-bit case of the vector compare operation was incorrectly incrementing by 8 bytes per iteration instead of 4 bytes. This caused the function to process only half of the intended elements. Cc: qemu-stable@nongnu.org Fixes: 9622c697d1 (tcg: Add gvec compare with immediate and scalar operand) Signed-off-by: TANG Tiancheng Reviewed-by: Liu Zhiwei Reviewed-by: Richard Henderson Message-ID: <20240904142739.854-2-zhiwei_liu@linux.alibaba.com> Signed-off-by: Richard Henderson Reviewed-by: Philippe Mathieu-Daudé --- tcg/tcg-op-gvec.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tcg') diff --git a/tcg/tcg-op-gvec.c b/tcg/tcg-op-gvec.c index 0308732d9b..78ee1ced80 100644 --- a/tcg/tcg-op-gvec.c +++ b/tcg/tcg-op-gvec.c @@ -3939,7 +3939,7 @@ void tcg_gen_gvec_cmps(TCGCond cond, unsigned vece, uint32_t dofs, uint32_t i; tcg_gen_extrl_i64_i32(t1, c); - for (i = 0; i < oprsz; i += 8) { + for (i = 0; i < oprsz; i += 4) { tcg_gen_ld_i32(t0, tcg_env, aofs + i); tcg_gen_negsetcond_i32(cond, t0, t0, t1); tcg_gen_st_i32(t0, tcg_env, dofs + i); -- cgit v1.2.3 From 8dd2ea75154d406c1d6e5fed797eef522c293a86 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 7 Sep 2024 16:27:48 -0700 Subject: tcg: Export vec_gen_6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add declaration to tcg-internal.h, making it available for use from tcg backend vector expanders. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/tcg-internal.h | 2 ++ tcg/tcg-op-vec.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'tcg') diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index d18f49f5d3..8099248076 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -102,5 +102,7 @@ TCGOp *tcg_gen_op6(TCGOpcode, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg, TCGArg); void vec_gen_2(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg); void vec_gen_3(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg, TCGArg); void vec_gen_4(TCGOpcode, TCGType, unsigned, TCGArg, TCGArg, TCGArg, TCGArg); +void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, + TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e); #endif /* TCG_INTERNAL_H */ diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index 84af210bc0..d4bb4aee74 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -172,8 +172,8 @@ void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, op->args[3] = c; } -static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, - TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) +void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, + TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) { TCGOp *op = tcg_emit_op(opc, 6); TCGOP_VECL(op) = type - TCG_TYPE_V64; -- cgit v1.2.3 From bc97b3ad313baf01ff7800c472c2a6931ff71dfb Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 6 Sep 2024 15:56:57 -0700 Subject: tcg/i386: Split out tcg_out_vex_modrm_type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Helper function to handle setting of VEXL based on the type of the operation. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/i386/tcg-target.c.inc | 38 +++++++++++++++----------------------- 1 file changed, 15 insertions(+), 23 deletions(-) (limited to 'tcg') diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index 9a54ef7f8d..af71a397b1 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -711,6 +711,15 @@ static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm) tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); } +static void tcg_out_vex_modrm_type(TCGContext *s, int opc, + int r, int v, int rm, TCGType type) +{ + if (type == TCG_TYPE_V256) { + opc |= P_VEXL; + } + tcg_out_vex_modrm(s, opc, r, v, rm); +} + /* Output an opcode with a full "rm + (index< Date: Fri, 6 Sep 2024 18:45:00 -0700 Subject: tcg/i386: Do not expand cmp_vec early Move most of expansion to opcode generation, leaving the conversion of unsigned to signed to be done in the early phase. Small inefficiencies, but not incorrect results, are introduced until cmpsel_vec is converted in the next patch. Signed-off-by: Richard Henderson --- tcg/i386/tcg-target.c.inc | 223 +++++++++++++++++++++------------------------- 1 file changed, 100 insertions(+), 123 deletions(-) (limited to 'tcg') diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index af71a397b1..278e567b56 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -3029,6 +3029,92 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, #undef OP_32_64 } +static int const umin_insn[4] = { + OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ +}; + +static int const umax_insn[4] = { + OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_VPMAXUQ +}; + +static bool tcg_out_cmp_vec_noinv(TCGContext *s, TCGType type, unsigned vece, + TCGReg v0, TCGReg v1, TCGReg v2, TCGCond cond) +{ + static int const cmpeq_insn[4] = { + OPC_PCMPEQB, OPC_PCMPEQW, OPC_PCMPEQD, OPC_PCMPEQQ + }; + static int const cmpgt_insn[4] = { + OPC_PCMPGTB, OPC_PCMPGTW, OPC_PCMPGTD, OPC_PCMPGTQ + }; + + enum { + NEED_INV = 1, + NEED_SWAP = 2, + NEED_UMIN = 4, + NEED_UMAX = 8, + INVALID = 16, + }; + static const uint8_t cond_fixup[16] = { + [0 ... 15] = INVALID, + [TCG_COND_EQ] = 0, + [TCG_COND_GT] = 0, + [TCG_COND_NE] = NEED_INV, + [TCG_COND_LE] = NEED_INV, + [TCG_COND_LT] = NEED_SWAP, + [TCG_COND_GE] = NEED_SWAP | NEED_INV, + [TCG_COND_LEU] = NEED_UMIN, + [TCG_COND_GTU] = NEED_UMIN | NEED_INV, + [TCG_COND_GEU] = NEED_UMAX, + [TCG_COND_LTU] = NEED_UMAX | NEED_INV, + }; + int fixup = cond_fixup[cond]; + + assert(!(fixup & INVALID)); + + if (fixup & NEED_INV) { + cond = tcg_invert_cond(cond); + } + + if (fixup & NEED_SWAP) { + TCGReg swap = v1; + v1 = v2; + v2 = swap; + cond = tcg_swap_cond(cond); + } + + if (fixup & (NEED_UMIN | NEED_UMAX)) { + int op = (fixup & NEED_UMIN ? umin_insn[vece] : umax_insn[vece]); + + /* avx2 does not have 64-bit min/max; adjusted during expand. */ + assert(vece <= MO_32); + + tcg_out_vex_modrm_type(s, op, TCG_TMP_VEC, v1, v2, type); + v2 = TCG_TMP_VEC; + cond = TCG_COND_EQ; + } + + switch (cond) { + case TCG_COND_EQ: + tcg_out_vex_modrm_type(s, cmpeq_insn[vece], v0, v1, v2, type); + break; + case TCG_COND_GT: + tcg_out_vex_modrm_type(s, cmpgt_insn[vece], v0, v1, v2, type); + break; + default: + g_assert_not_reached(); + } + return fixup & NEED_INV; +} + +static void tcg_out_cmp_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg v0, TCGReg v1, TCGReg v2, TCGCond cond) +{ + if (tcg_out_cmp_vec_noinv(s, type, vece, v0, v1, v2, cond)) { + tcg_out_dupi_vec(s, type, vece, TCG_TMP_VEC, -1); + tcg_out_vex_modrm_type(s, OPC_PXOR, v0, v0, TCG_TMP_VEC, type); + } +} + static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg args[TCG_MAX_OP_ARGS], @@ -3058,12 +3144,6 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, static int const shift_imm_insn[4] = { OPC_UD2, OPC_PSHIFTW_Ib, OPC_PSHIFTD_Ib, OPC_PSHIFTQ_Ib }; - static int const cmpeq_insn[4] = { - OPC_PCMPEQB, OPC_PCMPEQW, OPC_PCMPEQD, OPC_PCMPEQQ - }; - static int const cmpgt_insn[4] = { - OPC_PCMPGTB, OPC_PCMPGTW, OPC_PCMPGTD, OPC_PCMPGTQ - }; static int const punpckl_insn[4] = { OPC_PUNPCKLBW, OPC_PUNPCKLWD, OPC_PUNPCKLDQ, OPC_PUNPCKLQDQ }; @@ -3082,12 +3162,6 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, static int const smax_insn[4] = { OPC_PMAXSB, OPC_PMAXSW, OPC_PMAXSD, OPC_VPMAXSQ }; - static int const umin_insn[4] = { - OPC_PMINUB, OPC_PMINUW, OPC_PMINUD, OPC_VPMINUQ - }; - static int const umax_insn[4] = { - OPC_PMAXUB, OPC_PMAXUW, OPC_PMAXUD, OPC_VPMAXUQ - }; static int const rotlv_insn[4] = { OPC_UD2, OPC_UD2, OPC_VPROLVD, OPC_VPROLVQ }; @@ -3243,15 +3317,8 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_cmp_vec: - sub = args[3]; - if (sub == TCG_COND_EQ) { - insn = cmpeq_insn[vece]; - } else if (sub == TCG_COND_GT) { - insn = cmpgt_insn[vece]; - } else { - g_assert_not_reached(); - } - goto gen_simd; + tcg_out_cmp_vec(s, type, vece, a0, a1, a2, args[3]); + break; case INDEX_op_andc_vec: insn = OPC_PANDN; @@ -3971,88 +4038,19 @@ static void expand_vec_mul(TCGType type, unsigned vece, } } -static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, - TCGv_vec v1, TCGv_vec v2, TCGCond cond) +static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, + TCGv_vec v1, TCGv_vec v2, TCGCond cond) { - enum { - NEED_INV = 1, - NEED_SWAP = 2, - NEED_BIAS = 4, - NEED_UMIN = 8, - NEED_UMAX = 16, - }; - TCGv_vec t1, t2, t3; - uint8_t fixup; - - switch (cond) { - case TCG_COND_EQ: - case TCG_COND_GT: - fixup = 0; - break; - case TCG_COND_NE: - case TCG_COND_LE: - fixup = NEED_INV; - break; - case TCG_COND_LT: - fixup = NEED_SWAP; - break; - case TCG_COND_GE: - fixup = NEED_SWAP | NEED_INV; - break; - case TCG_COND_LEU: - if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) { - fixup = NEED_UMIN; - } else { - fixup = NEED_BIAS | NEED_INV; - } - break; - case TCG_COND_GTU: - if (tcg_can_emit_vec_op(INDEX_op_umin_vec, type, vece)) { - fixup = NEED_UMIN | NEED_INV; - } else { - fixup = NEED_BIAS; - } - break; - case TCG_COND_GEU: - if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) { - fixup = NEED_UMAX; - } else { - fixup = NEED_BIAS | NEED_SWAP | NEED_INV; - } - break; - case TCG_COND_LTU: - if (tcg_can_emit_vec_op(INDEX_op_umax_vec, type, vece)) { - fixup = NEED_UMAX | NEED_INV; - } else { - fixup = NEED_BIAS | NEED_SWAP; - } - break; - default: - g_assert_not_reached(); - } - - if (fixup & NEED_INV) { - cond = tcg_invert_cond(cond); - } - if (fixup & NEED_SWAP) { - t1 = v1, v1 = v2, v2 = t1; - cond = tcg_swap_cond(cond); - } + /* + * Without AVX512, there are no 64-bit unsigned comparisons. + * We must bias the inputs so that they become signed. + * All other swapping and inversion are handled during code generation. + */ + if (vece == MO_64 && is_unsigned_cond(cond)) { + TCGv_vec t1 = tcg_temp_new_vec(type); + TCGv_vec t2 = tcg_temp_new_vec(type); + TCGv_vec t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1)); - t1 = t2 = NULL; - if (fixup & (NEED_UMIN | NEED_UMAX)) { - t1 = tcg_temp_new_vec(type); - if (fixup & NEED_UMIN) { - tcg_gen_umin_vec(vece, t1, v1, v2); - } else { - tcg_gen_umax_vec(vece, t1, v1, v2); - } - v2 = t1; - cond = TCG_COND_EQ; - } else if (fixup & NEED_BIAS) { - t1 = tcg_temp_new_vec(type); - t2 = tcg_temp_new_vec(type); - t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1)); tcg_gen_sub_vec(vece, t1, v1, t3); tcg_gen_sub_vec(vece, t2, v2, t3); v1 = t1; @@ -4060,26 +4058,9 @@ static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, cond = tcg_signed_cond(cond); } - tcg_debug_assert(cond == TCG_COND_EQ || cond == TCG_COND_GT); /* Expand directly; do not recurse. */ vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond); - - if (t1) { - tcg_temp_free_vec(t1); - if (t2) { - tcg_temp_free_vec(t2); - } - } - return fixup & NEED_INV; -} - -static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, - TCGv_vec v1, TCGv_vec v2, TCGCond cond) -{ - if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) { - tcg_gen_not_vec(vece, v0, v0); - } } static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0, @@ -4088,11 +4069,7 @@ static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0, { TCGv_vec t = tcg_temp_new_vec(type); - if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) { - /* Invert the sense of the compare by swapping arguments. */ - TCGv_vec x; - x = v3, v3 = v4, v4 = x; - } + expand_vec_cmp(type, vece, t, c1, c2, cond); vec_gen_4(INDEX_op_x86_vpblendvb_vec, type, vece, tcgv_vec_arg(v0), tcgv_vec_arg(v4), tcgv_vec_arg(v3), tcgv_vec_arg(t)); -- cgit v1.2.3 From db4121d20715caa74c5863da64754bdf4baeeda6 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 6 Sep 2024 20:32:38 -0700 Subject: tcg/i386: Do not expand cmpsel_vec early Expand during output instead of during opcode generation. Remove x86_vpblendvb_vec opcode, this this removes the only user. Signed-off-by: Richard Henderson --- tcg/i386/tcg-target-con-set.h | 1 + tcg/i386/tcg-target.c.inc | 82 ++++++++++++++++++++++++++----------------- tcg/i386/tcg-target.h | 2 +- tcg/i386/tcg-target.opc.h | 1 - 4 files changed, 52 insertions(+), 34 deletions(-) (limited to 'tcg') diff --git a/tcg/i386/tcg-target-con-set.h b/tcg/i386/tcg-target-con-set.h index e24241cfa2..da4411d96b 100644 --- a/tcg/i386/tcg-target-con-set.h +++ b/tcg/i386/tcg-target-con-set.h @@ -50,6 +50,7 @@ C_N1_I2(r, r, r) C_N1_I2(r, r, rW) C_O1_I3(x, 0, x, x) C_O1_I3(x, x, x, x) +C_O1_I4(x, x, x, x, x) C_O1_I4(r, r, reT, r, 0) C_O1_I4(r, r, r, ri, ri) C_O2_I1(r, r, L) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index 278e567b56..a04dc7d270 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -3115,6 +3115,19 @@ static void tcg_out_cmp_vec(TCGContext *s, TCGType type, unsigned vece, } } +static void tcg_out_cmpsel_vec(TCGContext *s, TCGType type, unsigned vece, + TCGReg v0, TCGReg c1, TCGReg c2, + TCGReg v3, TCGReg v4, TCGCond cond) +{ + if (tcg_out_cmp_vec_noinv(s, type, vece, TCG_TMP_VEC, c1, c2, cond)) { + TCGReg swap = v3; + v3 = v4; + v4 = swap; + } + tcg_out_vex_modrm_type(s, OPC_VPBLENDVB, v0, v4, v3, type); + tcg_out8(s, (TCG_TMP_VEC - TCG_REG_XMM0) << 4); +} + static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg args[TCG_MAX_OP_ARGS], @@ -3320,6 +3333,11 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_cmp_vec(s, type, vece, a0, a1, a2, args[3]); break; + case INDEX_op_cmpsel_vec: + tcg_out_cmpsel_vec(s, type, vece, a0, a1, a2, + args[3], args[4], args[5]); + break; + case INDEX_op_andc_vec: insn = OPC_PANDN; tcg_out_vex_modrm_type(s, insn, a0, a2, a1, type); @@ -3431,11 +3449,6 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out8(s, sub); break; - case INDEX_op_x86_vpblendvb_vec: - tcg_out_vex_modrm_type(s, OPC_VPBLENDVB, a0, a1, a2, type); - tcg_out8(s, args[3] << 4); - break; - case INDEX_op_x86_psrldq_vec: tcg_out_vex_modrm(s, OPC_GRP14, 3, a0, a1); tcg_out8(s, a2); @@ -3701,8 +3714,9 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) return C_O1_I3(x, 0, x, x); case INDEX_op_bitsel_vec: - case INDEX_op_x86_vpblendvb_vec: return C_O1_I3(x, x, x, x); + case INDEX_op_cmpsel_vec: + return C_O1_I4(x, x, x, x, x); default: g_assert_not_reached(); @@ -4038,8 +4052,8 @@ static void expand_vec_mul(TCGType type, unsigned vece, } } -static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, - TCGv_vec v1, TCGv_vec v2, TCGCond cond) +static TCGCond expand_vec_cond(TCGType type, unsigned vece, + TCGArg *a1, TCGArg *a2, TCGCond cond) { /* * Without AVX512, there are no 64-bit unsigned comparisons. @@ -4047,46 +4061,50 @@ static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, * All other swapping and inversion are handled during code generation. */ if (vece == MO_64 && is_unsigned_cond(cond)) { + TCGv_vec v1 = temp_tcgv_vec(arg_temp(*a1)); + TCGv_vec v2 = temp_tcgv_vec(arg_temp(*a2)); TCGv_vec t1 = tcg_temp_new_vec(type); TCGv_vec t2 = tcg_temp_new_vec(type); TCGv_vec t3 = tcg_constant_vec(type, vece, 1ull << ((8 << vece) - 1)); tcg_gen_sub_vec(vece, t1, v1, t3); tcg_gen_sub_vec(vece, t2, v2, t3); - v1 = t1; - v2 = t2; + *a1 = tcgv_vec_arg(t1); + *a2 = tcgv_vec_arg(t2); cond = tcg_signed_cond(cond); } + return cond; +} +static void expand_vec_cmp(TCGType type, unsigned vece, TCGArg a0, + TCGArg a1, TCGArg a2, TCGCond cond) +{ + cond = expand_vec_cond(type, vece, &a1, &a2, cond); /* Expand directly; do not recurse. */ - vec_gen_4(INDEX_op_cmp_vec, type, vece, - tcgv_vec_arg(v0), tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond); + vec_gen_4(INDEX_op_cmp_vec, type, vece, a0, a1, a2, cond); } -static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0, - TCGv_vec c1, TCGv_vec c2, - TCGv_vec v3, TCGv_vec v4, TCGCond cond) +static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGArg a0, + TCGArg a1, TCGArg a2, + TCGArg a3, TCGArg a4, TCGCond cond) { - TCGv_vec t = tcg_temp_new_vec(type); - - expand_vec_cmp(type, vece, t, c1, c2, cond); - vec_gen_4(INDEX_op_x86_vpblendvb_vec, type, vece, - tcgv_vec_arg(v0), tcgv_vec_arg(v4), - tcgv_vec_arg(v3), tcgv_vec_arg(t)); - tcg_temp_free_vec(t); + cond = expand_vec_cond(type, vece, &a1, &a2, cond); + /* Expand directly; do not recurse. */ + vec_gen_6(INDEX_op_cmpsel_vec, type, vece, a0, a1, a2, a3, a4, cond); } void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, TCGArg a0, ...) { va_list va; - TCGArg a2; - TCGv_vec v0, v1, v2, v3, v4; + TCGArg a1, a2, a3, a4, a5; + TCGv_vec v0, v1, v2; va_start(va, a0); - v0 = temp_tcgv_vec(arg_temp(a0)); - v1 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); + a1 = va_arg(va, TCGArg); a2 = va_arg(va, TCGArg); + v0 = temp_tcgv_vec(arg_temp(a0)); + v1 = temp_tcgv_vec(arg_temp(a1)); switch (opc) { case INDEX_op_shli_vec: @@ -4122,15 +4140,15 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, break; case INDEX_op_cmp_vec: - v2 = temp_tcgv_vec(arg_temp(a2)); - expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); + a3 = va_arg(va, TCGArg); + expand_vec_cmp(type, vece, a0, a1, a2, a3); break; case INDEX_op_cmpsel_vec: - v2 = temp_tcgv_vec(arg_temp(a2)); - v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); - v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); - expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg)); + a3 = va_arg(va, TCGArg); + a4 = va_arg(va, TCGArg); + a5 = va_arg(va, TCGArg); + expand_vec_cmpsel(type, vece, a0, a1, a2, a3, a4, a5); break; default: diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index 2f67a97e05..342be30c4c 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -223,7 +223,7 @@ typedef enum { #define TCG_TARGET_HAS_sat_vec 1 #define TCG_TARGET_HAS_minmax_vec 1 #define TCG_TARGET_HAS_bitsel_vec have_avx512vl -#define TCG_TARGET_HAS_cmpsel_vec -1 +#define TCG_TARGET_HAS_cmpsel_vec 1 #define TCG_TARGET_HAS_tst_vec 0 #define TCG_TARGET_deposit_i32_valid(ofs, len) \ diff --git a/tcg/i386/tcg-target.opc.h b/tcg/i386/tcg-target.opc.h index b5f403e35e..4ffc084bda 100644 --- a/tcg/i386/tcg-target.opc.h +++ b/tcg/i386/tcg-target.opc.h @@ -25,7 +25,6 @@ */ DEF(x86_shufps_vec, 1, 2, 1, IMPLVEC) -DEF(x86_vpblendvb_vec, 1, 3, 0, IMPLVEC) DEF(x86_blend_vec, 1, 2, 1, IMPLVEC) DEF(x86_packss_vec, 1, 2, 0, IMPLVEC) DEF(x86_packus_vec, 1, 2, 0, IMPLVEC) -- cgit v1.2.3 From 2cd118ca4ada265cf5b3ceab807dda7a07437d43 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 10 Sep 2024 01:19:28 +0000 Subject: tcg/ppc: Do not expand cmp_vec early Move expansion to opcode generation. Signed-off-by: Richard Henderson --- tcg/ppc/tcg-target.c.inc | 169 +++++++++++++++++++++++++---------------------- 1 file changed, 90 insertions(+), 79 deletions(-) (limited to 'tcg') diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 3553a47ba9..497e130581 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -3567,12 +3567,13 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_usadd_vec: case INDEX_op_ussub_vec: return vece <= MO_32; - case INDEX_op_cmp_vec: case INDEX_op_shli_vec: case INDEX_op_shri_vec: case INDEX_op_sari_vec: case INDEX_op_rotli_vec: return vece <= MO_32 || have_isa_2_07 ? -1 : 0; + case INDEX_op_cmp_vec: + return vece <= MO_32 || have_isa_2_07 ? 1 : 0; case INDEX_op_neg_vec: return vece >= MO_32 && have_isa_3_00; case INDEX_op_mul_vec: @@ -3713,6 +3714,90 @@ static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece, return true; } +static void tcg_out_not_vec(TCGContext *s, TCGReg a0, TCGReg a1) +{ + tcg_out32(s, VNOR | VRT(a0) | VRA(a1) | VRB(a1)); +} + +static bool tcg_out_cmp_vec_noinv(TCGContext *s, unsigned vece, TCGReg a0, + TCGReg a1, TCGReg a2, TCGCond cond) +{ + static const uint32_t + eq_op[4] = { VCMPEQUB, VCMPEQUH, VCMPEQUW, VCMPEQUD }, + ne_op[4] = { VCMPNEB, VCMPNEH, VCMPNEW, 0 }, + gts_op[4] = { VCMPGTSB, VCMPGTSH, VCMPGTSW, VCMPGTSD }, + gtu_op[4] = { VCMPGTUB, VCMPGTUH, VCMPGTUW, VCMPGTUD }; + uint32_t insn; + + bool need_swap = false, need_inv = false; + + tcg_debug_assert(vece <= MO_32 || have_isa_2_07); + + switch (cond) { + case TCG_COND_EQ: + case TCG_COND_GT: + case TCG_COND_GTU: + break; + case TCG_COND_NE: + if (have_isa_3_00 && vece <= MO_32) { + break; + } + /* fall through */ + case TCG_COND_LE: + case TCG_COND_LEU: + need_inv = true; + break; + case TCG_COND_LT: + case TCG_COND_LTU: + need_swap = true; + break; + case TCG_COND_GE: + case TCG_COND_GEU: + need_swap = need_inv = true; + break; + default: + g_assert_not_reached(); + } + + if (need_inv) { + cond = tcg_invert_cond(cond); + } + if (need_swap) { + TCGReg swap = a1; + a1 = a2; + a2 = swap; + cond = tcg_swap_cond(cond); + } + + switch (cond) { + case TCG_COND_EQ: + insn = eq_op[vece]; + break; + case TCG_COND_NE: + insn = ne_op[vece]; + break; + case TCG_COND_GT: + insn = gts_op[vece]; + break; + case TCG_COND_GTU: + insn = gtu_op[vece]; + break; + default: + g_assert_not_reached(); + } + tcg_out32(s, insn | VRT(a0) | VRA(a1) | VRB(a2)); + + return need_inv; +} + +static void tcg_out_cmp_vec(TCGContext *s, unsigned vece, TCGReg a0, + TCGReg a1, TCGReg a2, TCGCond cond) +{ + if (tcg_out_cmp_vec_noinv(s, vece, a0, a1, a2, cond)) { + tcg_out_not_vec(s, a0, a0); + } +} + static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg args[TCG_MAX_OP_ARGS], @@ -3723,10 +3808,6 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, sub_op[4] = { VSUBUBM, VSUBUHM, VSUBUWM, VSUBUDM }, mul_op[4] = { 0, 0, VMULUWM, VMULLD }, neg_op[4] = { 0, 0, VNEGW, VNEGD }, - eq_op[4] = { VCMPEQUB, VCMPEQUH, VCMPEQUW, VCMPEQUD }, - ne_op[4] = { VCMPNEB, VCMPNEH, VCMPNEW, 0 }, - gts_op[4] = { VCMPGTSB, VCMPGTSH, VCMPGTSW, VCMPGTSD }, - gtu_op[4] = { VCMPGTUB, VCMPGTUH, VCMPGTUW, VCMPGTUD }, ssadd_op[4] = { VADDSBS, VADDSHS, VADDSWS, 0 }, usadd_op[4] = { VADDUBS, VADDUHS, VADDUWS, 0 }, sssub_op[4] = { VSUBSBS, VSUBSHS, VSUBSWS, 0 }, @@ -3820,9 +3901,8 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, insn = VANDC; break; case INDEX_op_not_vec: - insn = VNOR; - a2 = a1; - break; + tcg_out_not_vec(s, a0, a1); + return; case INDEX_op_orc_vec: insn = VORC; break; @@ -3837,23 +3917,8 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_cmp_vec: - switch (args[3]) { - case TCG_COND_EQ: - insn = eq_op[vece]; - break; - case TCG_COND_NE: - insn = ne_op[vece]; - break; - case TCG_COND_GT: - insn = gts_op[vece]; - break; - case TCG_COND_GTU: - insn = gtu_op[vece]; - break; - default: - g_assert_not_reached(); - } - break; + tcg_out_cmp_vec(s, vece, a0, a1, a2, args[3]); + return; case INDEX_op_bitsel_vec: tcg_out32(s, XXSEL | VRT(a0) | VRC(a1) | VRB(a2) | VRA(args[3])); @@ -3921,56 +3986,6 @@ static void expand_vec_shi(TCGType type, unsigned vece, TCGv_vec v0, tcgv_vec_arg(v1), tcgv_vec_arg(t1)); } -static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, - TCGv_vec v1, TCGv_vec v2, TCGCond cond) -{ - bool need_swap = false, need_inv = false; - - tcg_debug_assert(vece <= MO_32 || have_isa_2_07); - - switch (cond) { - case TCG_COND_EQ: - case TCG_COND_GT: - case TCG_COND_GTU: - break; - case TCG_COND_NE: - if (have_isa_3_00 && vece <= MO_32) { - break; - } - /* fall through */ - case TCG_COND_LE: - case TCG_COND_LEU: - need_inv = true; - break; - case TCG_COND_LT: - case TCG_COND_LTU: - need_swap = true; - break; - case TCG_COND_GE: - case TCG_COND_GEU: - need_swap = need_inv = true; - break; - default: - g_assert_not_reached(); - } - - if (need_inv) { - cond = tcg_invert_cond(cond); - } - if (need_swap) { - TCGv_vec t1; - t1 = v1, v1 = v2, v2 = t1; - cond = tcg_swap_cond(cond); - } - - vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0), - tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond); - - if (need_inv) { - tcg_gen_not_vec(vece, v0, v0); - } -} - static void expand_vec_mul(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec v1, TCGv_vec v2) { @@ -4045,10 +4060,6 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, case INDEX_op_rotli_vec: expand_vec_shi(type, vece, v0, v1, a2, INDEX_op_rotlv_vec); break; - case INDEX_op_cmp_vec: - v2 = temp_tcgv_vec(arg_temp(a2)); - expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); - break; case INDEX_op_mul_vec: v2 = temp_tcgv_vec(arg_temp(a2)); expand_vec_mul(type, vece, v0, v1, v2); -- cgit v1.2.3 From fcc54e7bf56ba627f9b6ac4a32c6b446d2591ccf Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 11 Sep 2024 01:01:36 +0000 Subject: tcg/s390x: Do not expand cmp_vec early Move expansion to opcode generation. Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 139 +++++++++++++++++++++------------------------ 1 file changed, 65 insertions(+), 74 deletions(-) (limited to 'tcg') diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index ad587325fc..23935fd0f0 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -2841,6 +2841,67 @@ static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece, tcg_out_insn(s, VRX, VLREP, dst, TCG_TMP0, TCG_REG_NONE, 0, MO_64); } +static bool tcg_out_cmp_vec_noinv(TCGContext *s, unsigned vece, TCGReg a0, + TCGReg a1, TCGReg a2, TCGCond cond) +{ + bool need_swap = false, need_inv = false; + + switch (cond) { + case TCG_COND_EQ: + case TCG_COND_GT: + case TCG_COND_GTU: + break; + case TCG_COND_NE: + case TCG_COND_LE: + case TCG_COND_LEU: + need_inv = true; + break; + case TCG_COND_LT: + case TCG_COND_LTU: + need_swap = true; + break; + case TCG_COND_GE: + case TCG_COND_GEU: + need_swap = need_inv = true; + break; + default: + g_assert_not_reached(); + } + + if (need_inv) { + cond = tcg_invert_cond(cond); + } + if (need_swap) { + TCGReg swap = a1; + a1 = a2; + a2 = swap; + cond = tcg_swap_cond(cond); + } + + switch (cond) { + case TCG_COND_EQ: + tcg_out_insn(s, VRRc, VCEQ, a0, a1, a2, vece); + break; + case TCG_COND_GT: + tcg_out_insn(s, VRRc, VCH, a0, a1, a2, vece); + break; + case TCG_COND_GTU: + tcg_out_insn(s, VRRc, VCHL, a0, a1, a2, vece); + break; + default: + g_assert_not_reached(); + } + return need_inv; +} + +static void tcg_out_cmp_vec(TCGContext *s, unsigned vece, TCGReg a0, + TCGReg a1, TCGReg a2, TCGCond cond) +{ + if (tcg_out_cmp_vec_noinv(s, vece, a0, a1, a2, cond)) { + tcg_out_insn(s, VRRc, VNO, a0, a0, a0, 0); + } +} + static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg args[TCG_MAX_OP_ARGS], @@ -2959,19 +3020,7 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, break; case INDEX_op_cmp_vec: - switch ((TCGCond)args[3]) { - case TCG_COND_EQ: - tcg_out_insn(s, VRRc, VCEQ, a0, a1, a2, vece); - break; - case TCG_COND_GT: - tcg_out_insn(s, VRRc, VCH, a0, a1, a2, vece); - break; - case TCG_COND_GTU: - tcg_out_insn(s, VRRc, VCHL, a0, a1, a2, vece); - break; - default: - g_assert_not_reached(); - } + tcg_out_cmp_vec(s, vece, a0, a1, a2, args[3]); break; case INDEX_op_s390_vuph_vec: @@ -3024,8 +3073,8 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_umax_vec: case INDEX_op_umin_vec: case INDEX_op_xor_vec: - return 1; case INDEX_op_cmp_vec: + return 1; case INDEX_op_cmpsel_vec: case INDEX_op_rotrv_vec: return -1; @@ -3039,68 +3088,14 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) } } -static bool expand_vec_cmp_noinv(TCGType type, unsigned vece, TCGv_vec v0, - TCGv_vec v1, TCGv_vec v2, TCGCond cond) -{ - bool need_swap = false, need_inv = false; - - switch (cond) { - case TCG_COND_EQ: - case TCG_COND_GT: - case TCG_COND_GTU: - break; - case TCG_COND_NE: - case TCG_COND_LE: - case TCG_COND_LEU: - need_inv = true; - break; - case TCG_COND_LT: - case TCG_COND_LTU: - need_swap = true; - break; - case TCG_COND_GE: - case TCG_COND_GEU: - need_swap = need_inv = true; - break; - default: - g_assert_not_reached(); - } - - if (need_inv) { - cond = tcg_invert_cond(cond); - } - if (need_swap) { - TCGv_vec t1; - t1 = v1, v1 = v2, v2 = t1; - cond = tcg_swap_cond(cond); - } - - vec_gen_4(INDEX_op_cmp_vec, type, vece, tcgv_vec_arg(v0), - tcgv_vec_arg(v1), tcgv_vec_arg(v2), cond); - - return need_inv; -} - -static void expand_vec_cmp(TCGType type, unsigned vece, TCGv_vec v0, - TCGv_vec v1, TCGv_vec v2, TCGCond cond) -{ - if (expand_vec_cmp_noinv(type, vece, v0, v1, v2, cond)) { - tcg_gen_not_vec(vece, v0, v0); - } -} - static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec c1, TCGv_vec c2, TCGv_vec v3, TCGv_vec v4, TCGCond cond) { TCGv_vec t = tcg_temp_new_vec(type); - if (expand_vec_cmp_noinv(type, vece, t, c1, c2, cond)) { - /* Invert the sense of the compare by swapping arguments. */ - tcg_gen_bitsel_vec(vece, v0, t, v4, v3); - } else { - tcg_gen_bitsel_vec(vece, v0, t, v3, v4); - } + tcg_gen_cmp_vec(cond, vece, t, c1, c2); + tcg_gen_bitsel_vec(vece, v0, t, v3, v4); tcg_temp_free_vec(t); } @@ -3153,10 +3148,6 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); switch (opc) { - case INDEX_op_cmp_vec: - expand_vec_cmp(type, vece, v0, v1, v2, va_arg(va, TCGArg)); - break; - case INDEX_op_cmpsel_vec: v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); -- cgit v1.2.3 From 141125e08cf422e22d40b0114a265c83d888767a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 6 Sep 2024 21:00:10 -0700 Subject: tcg/optimize: Fold movcond with true and false values identical MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold "x = cond ? y : y" to "x = y". Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/optimize.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'tcg') diff --git a/tcg/optimize.c b/tcg/optimize.c index ba16ec27e2..cf311790e0 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -1851,6 +1851,11 @@ static bool fold_movcond(OptContext *ctx, TCGOp *op) { int i; + /* If true and false values are the same, eliminate the cmp. */ + if (args_are_copies(op->args[3], op->args[4])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]); + } + /* * Canonicalize the "false" input reg to match the destination reg so * that the tcg backend can implement a "move if true" operation. -- cgit v1.2.3 From 1f106544fd87fd12566c8c5e12251067e2bc9f78 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 6 Sep 2024 12:22:41 -0700 Subject: tcg/optimize: Optimize cmp_vec and cmpsel_vec Place immediate values second in the comparison. Place destination matches first in the true/false values. All of this mirrors what we do for integer setcond and movcond. Signed-off-by: Richard Henderson --- tcg/optimize.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'tcg') diff --git a/tcg/optimize.c b/tcg/optimize.c index cf311790e0..f11f576fd4 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -2422,6 +2422,36 @@ static bool fold_setcond2(OptContext *ctx, TCGOp *op) return tcg_opt_gen_movi(ctx, op, op->args[0], i); } +static bool fold_cmp_vec(OptContext *ctx, TCGOp *op) +{ + /* Canonicalize the comparison to put immediate second. */ + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[3] = tcg_swap_cond(op->args[3]); + } + return false; +} + +static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op) +{ + /* If true and false values are the same, eliminate the cmp. */ + if (args_are_copies(op->args[3], op->args[4])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]); + } + + /* Canonicalize the comparison to put immediate second. */ + if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) { + op->args[5] = tcg_swap_cond(op->args[5]); + } + /* + * Canonicalize the "false" input reg to match the destination, + * so that the tcg backend can implement "move if true". + */ + if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) { + op->args[5] = tcg_invert_cond(op->args[5]); + } + return false; +} + static bool fold_sextract(OptContext *ctx, TCGOp *op) { uint64_t z_mask, s_mask, s_mask_old; @@ -2928,6 +2958,12 @@ void tcg_optimize(TCGContext *s) case INDEX_op_setcond2_i32: done = fold_setcond2(&ctx, op); break; + case INDEX_op_cmp_vec: + done = fold_cmp_vec(&ctx, op); + break; + case INDEX_op_cmpsel_vec: + done = fold_cmpsel_vec(&ctx, op); + break; CASE_OP_32_64(sextract): done = fold_sextract(&ctx, op); break; -- cgit v1.2.3 From e58b977238e3120cba3190e1107a111a44202a24 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 6 Sep 2024 22:30:01 -0700 Subject: tcg/optimize: Optimize bitsel_vec Fold matching true/false operands. Fold true/false operands with 0/-1 to simpler logicals. Signed-off-by: Richard Henderson --- tcg/optimize.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) (limited to 'tcg') diff --git a/tcg/optimize.c b/tcg/optimize.c index f11f576fd4..e9ef16b3c6 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -2737,6 +2737,61 @@ static bool fold_xor(OptContext *ctx, TCGOp *op) return fold_masks(ctx, op); } +static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op) +{ + /* If true and false values are the same, eliminate the cmp. */ + if (args_are_copies(op->args[2], op->args[3])) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]); + } + + if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) { + uint64_t tv = arg_info(op->args[2])->val; + uint64_t fv = arg_info(op->args[3])->val; + + if (tv == -1 && fv == 0) { + return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]); + } + if (tv == 0 && fv == -1) { + if (TCG_TARGET_HAS_not_vec) { + op->opc = INDEX_op_not_vec; + return fold_not(ctx, op); + } else { + op->opc = INDEX_op_xor_vec; + op->args[2] = arg_new_constant(ctx, -1); + return fold_xor(ctx, op); + } + } + } + if (arg_is_const(op->args[2])) { + uint64_t tv = arg_info(op->args[2])->val; + if (tv == -1) { + op->opc = INDEX_op_or_vec; + op->args[2] = op->args[3]; + return fold_or(ctx, op); + } + if (tv == 0 && TCG_TARGET_HAS_andc_vec) { + op->opc = INDEX_op_andc_vec; + op->args[2] = op->args[1]; + op->args[1] = op->args[3]; + return fold_andc(ctx, op); + } + } + if (arg_is_const(op->args[3])) { + uint64_t fv = arg_info(op->args[3])->val; + if (fv == 0) { + op->opc = INDEX_op_and_vec; + return fold_and(ctx, op); + } + if (fv == -1 && TCG_TARGET_HAS_orc_vec) { + op->opc = INDEX_op_orc_vec; + op->args[2] = op->args[1]; + op->args[1] = op->args[3]; + return fold_orc(ctx, op); + } + } + return false; +} + /* Propagate constants and copies, fold constant expressions. */ void tcg_optimize(TCGContext *s) { @@ -2964,6 +3019,9 @@ void tcg_optimize(TCGContext *s) case INDEX_op_cmpsel_vec: done = fold_cmpsel_vec(&ctx, op); break; + case INDEX_op_bitsel_vec: + done = fold_bitsel_vec(&ctx, op); + break; CASE_OP_32_64(sextract): done = fold_sextract(&ctx, op); break; -- cgit v1.2.3 From d8387f0ee0862985dc481db1fca78dced54e183a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 6 Sep 2024 22:01:12 -0700 Subject: tcg/i386: Optimize cmpsel with constant 0 operand 3. These can be simplified to and/andc, avoiding the load of the zero into a register. Signed-off-by: Richard Henderson --- tcg/i386/tcg-target-con-set.h | 2 +- tcg/i386/tcg-target-con-str.h | 1 + tcg/i386/tcg-target.c.inc | 32 +++++++++++++++++++++++++------- 3 files changed, 27 insertions(+), 8 deletions(-) (limited to 'tcg') diff --git a/tcg/i386/tcg-target-con-set.h b/tcg/i386/tcg-target-con-set.h index da4411d96b..06e6521001 100644 --- a/tcg/i386/tcg-target-con-set.h +++ b/tcg/i386/tcg-target-con-set.h @@ -50,7 +50,7 @@ C_N1_I2(r, r, r) C_N1_I2(r, r, rW) C_O1_I3(x, 0, x, x) C_O1_I3(x, x, x, x) -C_O1_I4(x, x, x, x, x) +C_O1_I4(x, x, x, xO, x) C_O1_I4(r, r, reT, r, 0) C_O1_I4(r, r, r, ri, ri) C_O2_I1(r, r, L) diff --git a/tcg/i386/tcg-target-con-str.h b/tcg/i386/tcg-target-con-str.h index cc22db227b..52142ab121 100644 --- a/tcg/i386/tcg-target-con-str.h +++ b/tcg/i386/tcg-target-con-str.h @@ -28,6 +28,7 @@ REGS('s', ALL_BYTEL_REGS & ~SOFTMMU_RESERVE_REGS) /* qemu_st8_i32 data */ */ CONST('e', TCG_CT_CONST_S32) CONST('I', TCG_CT_CONST_I32) +CONST('O', TCG_CT_CONST_ZERO) CONST('T', TCG_CT_CONST_TST) CONST('W', TCG_CT_CONST_WSZ) CONST('Z', TCG_CT_CONST_U32) diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index a04dc7d270..210389955d 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -133,6 +133,7 @@ static TCGReg tcg_target_call_oarg_reg(TCGCallReturnKind kind, int slot) #define TCG_CT_CONST_I32 0x400 #define TCG_CT_CONST_WSZ 0x800 #define TCG_CT_CONST_TST 0x1000 +#define TCG_CT_CONST_ZERO 0x2000 /* Registers used with L constraint, which are the first argument registers on x86_64, and two random call clobbered registers on @@ -226,6 +227,9 @@ static bool tcg_target_const_match(int64_t val, int ct, if ((ct & TCG_CT_CONST_WSZ) && val == (type == TCG_TYPE_I32 ? 32 : 64)) { return 1; } + if ((ct & TCG_CT_CONST_ZERO) && val == 0) { + return 1; + } return 0; } @@ -3119,13 +3123,27 @@ static void tcg_out_cmpsel_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg v0, TCGReg c1, TCGReg c2, TCGReg v3, TCGReg v4, TCGCond cond) { - if (tcg_out_cmp_vec_noinv(s, type, vece, TCG_TMP_VEC, c1, c2, cond)) { - TCGReg swap = v3; - v3 = v4; - v4 = swap; + bool inv = tcg_out_cmp_vec_noinv(s, type, vece, TCG_TMP_VEC, c1, c2, cond); + + /* + * Since XMM0 is 16, the only way we get 0 into V3 + * is via the constant zero constraint. + */ + if (!v3) { + if (inv) { + tcg_out_vex_modrm_type(s, OPC_PAND, v0, TCG_TMP_VEC, v4, type); + } else { + tcg_out_vex_modrm_type(s, OPC_PANDN, v0, TCG_TMP_VEC, v4, type); + } + } else { + if (inv) { + TCGReg swap = v3; + v3 = v4; + v4 = swap; + } + tcg_out_vex_modrm_type(s, OPC_VPBLENDVB, v0, v4, v3, type); + tcg_out8(s, (TCG_TMP_VEC - TCG_REG_XMM0) << 4); } - tcg_out_vex_modrm_type(s, OPC_VPBLENDVB, v0, v4, v3, type); - tcg_out8(s, (TCG_TMP_VEC - TCG_REG_XMM0) << 4); } static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, @@ -3716,7 +3734,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_bitsel_vec: return C_O1_I3(x, x, x, x); case INDEX_op_cmpsel_vec: - return C_O1_I4(x, x, x, x, x); + return C_O1_I4(x, x, x, xO, x); default: g_assert_not_reached(); -- cgit v1.2.3 From 717da87d38937e0573aeb5b9c24e3e296a24fab8 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 7 Sep 2024 08:41:39 -0700 Subject: tcg/i386: Implement cmp_vec with avx512 insns The sse/avx instruction set only has EQ and GT as direct comparisons. Other signed comparisons can be generated from swapping and inversion. However unsigned comparisons are not available and must be transformed to signed comparisons by biasing the inputs. The avx512 instruction set has a complete set of comparisons, with results placed into a predicate register. We can produce the normal cmp_vec result by using VPMOVM2*. Signed-off-by: Richard Henderson --- tcg/i386/tcg-target.c.inc | 64 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) (limited to 'tcg') diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index 210389955d..b1d642fc67 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -413,6 +413,14 @@ static bool tcg_target_const_match(int64_t val, int ct, #define OPC_UD2 (0x0b | P_EXT) #define OPC_VPBLENDD (0x02 | P_EXT3A | P_DATA16) #define OPC_VPBLENDVB (0x4c | P_EXT3A | P_DATA16) +#define OPC_VPCMPB (0x3f | P_EXT3A | P_DATA16 | P_EVEX) +#define OPC_VPCMPUB (0x3e | P_EXT3A | P_DATA16 | P_EVEX) +#define OPC_VPCMPW (0x3f | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPCMPUW (0x3e | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPCMPD (0x1f | P_EXT3A | P_DATA16 | P_EVEX) +#define OPC_VPCMPUD (0x1e | P_EXT3A | P_DATA16 | P_EVEX) +#define OPC_VPCMPQ (0x1f | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPCMPUQ (0x1e | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPINSRB (0x20 | P_EXT3A | P_DATA16) #define OPC_VPINSRW (0xc4 | P_EXT | P_DATA16) #define OPC_VBROADCASTSS (0x18 | P_EXT38 | P_DATA16) @@ -421,6 +429,10 @@ static bool tcg_target_const_match(int64_t val, int ct, #define OPC_VPBROADCASTW (0x79 | P_EXT38 | P_DATA16) #define OPC_VPBROADCASTD (0x58 | P_EXT38 | P_DATA16) #define OPC_VPBROADCASTQ (0x59 | P_EXT38 | P_DATA16) +#define OPC_VPMOVM2B (0x28 | P_EXT38 | P_SIMDF3 | P_EVEX) +#define OPC_VPMOVM2W (0x28 | P_EXT38 | P_SIMDF3 | P_VEXW | P_EVEX) +#define OPC_VPMOVM2D (0x38 | P_EXT38 | P_SIMDF3 | P_EVEX) +#define OPC_VPMOVM2Q (0x38 | P_EXT38 | P_SIMDF3 | P_VEXW | P_EVEX) #define OPC_VPERMQ (0x00 | P_EXT3A | P_DATA16 | P_VEXW) #define OPC_VPERM2I128 (0x46 | P_EXT3A | P_DATA16 | P_VEXL) #define OPC_VPROLVD (0x15 | P_EXT38 | P_DATA16 | P_EVEX) @@ -3110,9 +3122,59 @@ static bool tcg_out_cmp_vec_noinv(TCGContext *s, TCGType type, unsigned vece, return fixup & NEED_INV; } +static void tcg_out_cmp_vec_k1(TCGContext *s, TCGType type, unsigned vece, + TCGReg v1, TCGReg v2, TCGCond cond) +{ + static const int cmpm_insn[2][4] = { + { OPC_VPCMPB, OPC_VPCMPW, OPC_VPCMPD, OPC_VPCMPQ }, + { OPC_VPCMPUB, OPC_VPCMPUW, OPC_VPCMPUD, OPC_VPCMPUQ } + }; + static const int cond_ext[16] = { + [TCG_COND_EQ] = 0, + [TCG_COND_NE] = 4, + [TCG_COND_LT] = 1, + [TCG_COND_LTU] = 1, + [TCG_COND_LE] = 2, + [TCG_COND_LEU] = 2, + [TCG_COND_NEVER] = 3, + [TCG_COND_GE] = 5, + [TCG_COND_GEU] = 5, + [TCG_COND_GT] = 6, + [TCG_COND_GTU] = 6, + [TCG_COND_ALWAYS] = 7, + }; + + tcg_out_vex_modrm_type(s, cmpm_insn[is_unsigned_cond(cond)][vece], + /* k1 */ 1, v1, v2, type); + tcg_out8(s, cond_ext[cond]); +} + +static void tcg_out_k1_to_vec(TCGContext *s, TCGType type, + unsigned vece, TCGReg dest) +{ + static const int movm_insn[] = { + OPC_VPMOVM2B, OPC_VPMOVM2W, OPC_VPMOVM2D, OPC_VPMOVM2Q + }; + tcg_out_vex_modrm_type(s, movm_insn[vece], dest, 0, /* k1 */ 1, type); +} + static void tcg_out_cmp_vec(TCGContext *s, TCGType type, unsigned vece, TCGReg v0, TCGReg v1, TCGReg v2, TCGCond cond) { + /* + * With avx512, we have a complete set of comparisons into mask. + * Unless there's a single insn expansion for the comparision, + * expand via a mask in k1. + */ + if ((vece <= MO_16 ? have_avx512bw : have_avx512dq) + && cond != TCG_COND_EQ + && cond != TCG_COND_LT + && cond != TCG_COND_GT) { + tcg_out_cmp_vec_k1(s, type, vece, v1, v2, cond); + tcg_out_k1_to_vec(s, type, vece, v0); + return; + } + if (tcg_out_cmp_vec_noinv(s, type, vece, v0, v1, v2, cond)) { tcg_out_dupi_vec(s, type, vece, TCG_TMP_VEC, -1); tcg_out_vex_modrm_type(s, OPC_PXOR, v0, v0, TCG_TMP_VEC, type); @@ -4078,7 +4140,7 @@ static TCGCond expand_vec_cond(TCGType type, unsigned vece, * We must bias the inputs so that they become signed. * All other swapping and inversion are handled during code generation. */ - if (vece == MO_64 && is_unsigned_cond(cond)) { + if (vece == MO_64 && !have_avx512dq && is_unsigned_cond(cond)) { TCGv_vec v1 = temp_tcgv_vec(arg_temp(*a1)); TCGv_vec v2 = temp_tcgv_vec(arg_temp(*a2)); TCGv_vec t1 = tcg_temp_new_vec(type); -- cgit v1.2.3 From c044ec0d85cd94d1a986297c2e1f228408dddd76 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 7 Sep 2024 17:21:10 -0700 Subject: tcg/i386: Add predicate parameters to tcg_out_evex_opc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend tcg_out_evex_opc to handle the predicate and zero-merging parameters of the evex prefix. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/i386/tcg-target.c.inc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tcg') diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index b1d642fc67..f94a2a2385 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -674,7 +674,7 @@ static void tcg_out_vex_opc(TCGContext *s, int opc, int r, int v, } static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v, - int rm, int index) + int rm, int index, int aaa, bool z) { /* The entire 4-byte evex prefix; with R' and V' set. */ uint32_t p = 0x08041062; @@ -711,7 +711,9 @@ static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v, p = deposit32(p, 16, 2, pp); p = deposit32(p, 19, 4, ~v); p = deposit32(p, 23, 1, (opc & P_VEXW) != 0); + p = deposit32(p, 24, 3, aaa); p = deposit32(p, 29, 2, (opc & P_VEXL) != 0); + p = deposit32(p, 31, 1, z); tcg_out32(s, p); tcg_out8(s, opc); @@ -720,7 +722,7 @@ static void tcg_out_evex_opc(TCGContext *s, int opc, int r, int v, static void tcg_out_vex_modrm(TCGContext *s, int opc, int r, int v, int rm) { if (opc & P_EVEX) { - tcg_out_evex_opc(s, opc, r, v, rm, 0); + tcg_out_evex_opc(s, opc, r, v, rm, 0, 0, false); } else { tcg_out_vex_opc(s, opc, r, v, rm, 0); } -- cgit v1.2.3 From d58967490238f8d4c941102ade649314785d3f48 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 7 Sep 2024 17:24:57 -0700 Subject: tcg/i386: Implement cmpsel_vec with avx512 insns The avx512 vpblendm* instructions exactly implement cmpsel, using a predicate input. Of course this matches nicely with the avx512 predicate comparison instructions. Signed-off-by: Richard Henderson --- tcg/i386/tcg-target.c.inc | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) (limited to 'tcg') diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index f94a2a2385..d473dc7a5e 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -413,6 +413,10 @@ static bool tcg_target_const_match(int64_t val, int ct, #define OPC_UD2 (0x0b | P_EXT) #define OPC_VPBLENDD (0x02 | P_EXT3A | P_DATA16) #define OPC_VPBLENDVB (0x4c | P_EXT3A | P_DATA16) +#define OPC_VPBLENDMB (0x66 | P_EXT38 | P_DATA16 | P_EVEX) +#define OPC_VPBLENDMW (0x66 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPBLENDMD (0x64 | P_EXT38 | P_DATA16 | P_EVEX) +#define OPC_VPBLENDMQ (0x64 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) #define OPC_VPCMPB (0x3f | P_EXT3A | P_DATA16 | P_EVEX) #define OPC_VPCMPUB (0x3e | P_EXT3A | P_DATA16 | P_EVEX) #define OPC_VPCMPW (0x3f | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) @@ -738,6 +742,16 @@ static void tcg_out_vex_modrm_type(TCGContext *s, int opc, tcg_out_vex_modrm(s, opc, r, v, rm); } +static void tcg_out_evex_modrm_type(TCGContext *s, int opc, int r, int v, + int rm, int aaa, bool z, TCGType type) +{ + if (type == TCG_TYPE_V256) { + opc |= P_VEXL; + } + tcg_out_evex_opc(s, opc, r, v, rm, 0, aaa, z); + tcg_out8(s, 0xc0 | (LOWREGMASK(r) << 3) | LOWREGMASK(rm)); +} + /* Output an opcode with a full "rm + (index< Date: Sat, 7 Sep 2024 21:39:03 -0700 Subject: tcg/i386: Implement vector TST{EQ,NE} for avx512 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/i386/tcg-target.c.inc | 31 ++++++++++++++++++++++++++++--- tcg/i386/tcg-target.h | 2 +- 2 files changed, 29 insertions(+), 4 deletions(-) (limited to 'tcg') diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index d473dc7a5e..1bf50f1f62 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -462,6 +462,14 @@ static bool tcg_target_const_match(int64_t val, int ct, #define OPC_VPSRLVD (0x45 | P_EXT38 | P_DATA16) #define OPC_VPSRLVQ (0x45 | P_EXT38 | P_DATA16 | P_VEXW) #define OPC_VPTERNLOGQ (0x25 | P_EXT3A | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPTESTMB (0x26 | P_EXT38 | P_DATA16 | P_EVEX) +#define OPC_VPTESTMW (0x26 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPTESTMD (0x27 | P_EXT38 | P_DATA16 | P_EVEX) +#define OPC_VPTESTMQ (0x27 | P_EXT38 | P_DATA16 | P_VEXW | P_EVEX) +#define OPC_VPTESTNMB (0x26 | P_EXT38 | P_SIMDF3 | P_EVEX) +#define OPC_VPTESTNMW (0x26 | P_EXT38 | P_SIMDF3 | P_VEXW | P_EVEX) +#define OPC_VPTESTNMD (0x27 | P_EXT38 | P_SIMDF3 | P_EVEX) +#define OPC_VPTESTNMQ (0x27 | P_EXT38 | P_SIMDF3 | P_VEXW | P_EVEX) #define OPC_VZEROUPPER (0x77 | P_EXT) #define OPC_XCHG_ax_r32 (0x90) #define OPC_XCHG_EvGv (0x87) @@ -3145,6 +3153,13 @@ static void tcg_out_cmp_vec_k1(TCGContext *s, TCGType type, unsigned vece, { OPC_VPCMPB, OPC_VPCMPW, OPC_VPCMPD, OPC_VPCMPQ }, { OPC_VPCMPUB, OPC_VPCMPUW, OPC_VPCMPUD, OPC_VPCMPUQ } }; + static const int testm_insn[4] = { + OPC_VPTESTMB, OPC_VPTESTMW, OPC_VPTESTMD, OPC_VPTESTMQ + }; + static const int testnm_insn[4] = { + OPC_VPTESTNMB, OPC_VPTESTNMW, OPC_VPTESTNMD, OPC_VPTESTNMQ + }; + static const int cond_ext[16] = { [TCG_COND_EQ] = 0, [TCG_COND_NE] = 4, @@ -3160,9 +3175,19 @@ static void tcg_out_cmp_vec_k1(TCGContext *s, TCGType type, unsigned vece, [TCG_COND_ALWAYS] = 7, }; - tcg_out_vex_modrm_type(s, cmpm_insn[is_unsigned_cond(cond)][vece], - /* k1 */ 1, v1, v2, type); - tcg_out8(s, cond_ext[cond]); + switch (cond) { + case TCG_COND_TSTNE: + tcg_out_vex_modrm_type(s, testm_insn[vece], /* k1 */ 1, v1, v2, type); + break; + case TCG_COND_TSTEQ: + tcg_out_vex_modrm_type(s, testnm_insn[vece], /* k1 */ 1, v1, v2, type); + break; + default: + tcg_out_vex_modrm_type(s, cmpm_insn[is_unsigned_cond(cond)][vece], + /* k1 */ 1, v1, v2, type); + tcg_out8(s, cond_ext[cond]); + break; + } } static void tcg_out_k1_to_vec(TCGContext *s, TCGType type, diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index 342be30c4c..c68ac023d8 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -224,7 +224,7 @@ typedef enum { #define TCG_TARGET_HAS_minmax_vec 1 #define TCG_TARGET_HAS_bitsel_vec have_avx512vl #define TCG_TARGET_HAS_cmpsel_vec 1 -#define TCG_TARGET_HAS_tst_vec 0 +#define TCG_TARGET_HAS_tst_vec have_avx512bw #define TCG_TARGET_deposit_i32_valid(ofs, len) \ (((ofs) == 0 && ((len) == 8 || (len) == 16)) || \ -- cgit v1.2.3 From d0dabf9ec5bc8d4252cc7f166c28542a492eaaad Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 10 Sep 2024 04:43:54 +0000 Subject: tcg/ppc: Implement cmpsel_vec Do not allow cmpsel_vec to be expanded early, so that we can make the correct decision wrt the sense of the comparison. Signed-off-by: Richard Henderson --- tcg/ppc/tcg-target-con-set.h | 1 + tcg/ppc/tcg-target.c.inc | 60 ++++++++++++++++++++++++++++++++++++++------ tcg/ppc/tcg-target.h | 2 +- 3 files changed, 54 insertions(+), 9 deletions(-) (limited to 'tcg') diff --git a/tcg/ppc/tcg-target-con-set.h b/tcg/ppc/tcg-target-con-set.h index 9f99bde505..e7ba00c248 100644 --- a/tcg/ppc/tcg-target-con-set.h +++ b/tcg/ppc/tcg-target-con-set.h @@ -33,6 +33,7 @@ C_O1_I2(r, r, rU) C_O1_I2(r, r, rZW) C_O1_I2(v, v, v) C_O1_I3(v, v, v, v) +C_O1_I4(v, v, v, v, v) C_O1_I4(r, r, rC, rZ, rZ) C_O1_I4(r, r, r, ri, ri) C_O2_I1(r, r, r) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 497e130581..9d07b4d8e6 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -3573,6 +3573,7 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_rotli_vec: return vece <= MO_32 || have_isa_2_07 ? -1 : 0; case INDEX_op_cmp_vec: + case INDEX_op_cmpsel_vec: return vece <= MO_32 || have_isa_2_07 ? 1 : 0; case INDEX_op_neg_vec: return vece >= MO_32 && have_isa_3_00; @@ -3719,6 +3720,33 @@ static void tcg_out_not_vec(TCGContext *s, TCGReg a0, TCGReg a1) tcg_out32(s, VNOR | VRT(a0) | VRA(a1) | VRB(a1)); } +static void tcg_out_or_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, VOR | VRT(a0) | VRA(a1) | VRB(a2)); +} + +static void tcg_out_and_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, VAND | VRT(a0) | VRA(a1) | VRB(a2)); +} + +static void tcg_out_andc_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, VANDC | VRT(a0) | VRA(a1) | VRB(a2)); +} + +static void tcg_out_bitsel_vec(TCGContext *s, TCGReg d, + TCGReg c, TCGReg t, TCGReg f) +{ + if (TCG_TARGET_HAS_bitsel_vec) { + tcg_out32(s, XXSEL | VRT(d) | VRC(c) | VRB(t) | VRA(f)); + } else { + tcg_out_and_vec(s, TCG_VEC_TMP2, t, c); + tcg_out_andc_vec(s, d, f, c); + tcg_out_or_vec(s, d, d, TCG_VEC_TMP2); + } +} + static bool tcg_out_cmp_vec_noinv(TCGContext *s, unsigned vece, TCGReg a0, TCGReg a1, TCGReg a2, TCGCond cond) { @@ -3798,6 +3826,18 @@ static void tcg_out_cmp_vec(TCGContext *s, unsigned vece, TCGReg a0, } } +static void tcg_out_cmpsel_vec(TCGContext *s, unsigned vece, TCGReg a0, + TCGReg c1, TCGReg c2, TCGReg v3, TCGReg v4, + TCGCond cond) +{ + if (tcg_out_cmp_vec_noinv(s, vece, TCG_VEC_TMP1, c1, c2, cond)) { + TCGReg swap = v3; + v3 = v4; + v4 = swap; + } + tcg_out_bitsel_vec(s, a0, TCG_VEC_TMP1, v3, v4); +} + static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg args[TCG_MAX_OP_ARGS], @@ -3889,17 +3929,17 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, insn = sarv_op[vece]; break; case INDEX_op_and_vec: - insn = VAND; - break; + tcg_out_and_vec(s, a0, a1, a2); + return; case INDEX_op_or_vec: - insn = VOR; - break; + tcg_out_or_vec(s, a0, a1, a2); + return; case INDEX_op_xor_vec: insn = VXOR; break; case INDEX_op_andc_vec: - insn = VANDC; - break; + tcg_out_andc_vec(s, a0, a1, a2); + return; case INDEX_op_not_vec: tcg_out_not_vec(s, a0, a1); return; @@ -3919,9 +3959,11 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, case INDEX_op_cmp_vec: tcg_out_cmp_vec(s, vece, a0, a1, a2, args[3]); return; - + case INDEX_op_cmpsel_vec: + tcg_out_cmpsel_vec(s, vece, a0, a1, a2, args[3], args[4], args[5]); + return; case INDEX_op_bitsel_vec: - tcg_out32(s, XXSEL | VRT(a0) | VRC(a1) | VRB(a2) | VRA(args[3])); + tcg_out_bitsel_vec(s, a0, a1, a2, args[3]); return; case INDEX_op_dup2_vec: @@ -4287,6 +4329,8 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_bitsel_vec: case INDEX_op_ppc_msum_vec: return C_O1_I3(v, v, v, v); + case INDEX_op_cmpsel_vec: + return C_O1_I4(v, v, v, v, v); default: g_assert_not_reached(); diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h index e154fb14df..0b2171d38c 100644 --- a/tcg/ppc/tcg-target.h +++ b/tcg/ppc/tcg-target.h @@ -172,7 +172,7 @@ typedef enum { #define TCG_TARGET_HAS_sat_vec 1 #define TCG_TARGET_HAS_minmax_vec 1 #define TCG_TARGET_HAS_bitsel_vec have_vsx -#define TCG_TARGET_HAS_cmpsel_vec 0 +#define TCG_TARGET_HAS_cmpsel_vec 1 #define TCG_TARGET_HAS_tst_vec 0 #define TCG_TARGET_DEFAULT_MO (0) -- cgit v1.2.3 From ce8e5f2f2f585fbbf507d96477b7dd75bf9182eb Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 10 Sep 2024 05:34:56 +0000 Subject: tcg/ppc: Optimize cmpsel with constant 0/-1 arguments These can be simplified to and/or/andc/orc, avoiding the load of the constantinto a register. Signed-off-by: Richard Henderson --- tcg/ppc/tcg-target-con-set.h | 2 +- tcg/ppc/tcg-target.c.inc | 43 ++++++++++++++++++++++++++++++++----------- 2 files changed, 33 insertions(+), 12 deletions(-) (limited to 'tcg') diff --git a/tcg/ppc/tcg-target-con-set.h b/tcg/ppc/tcg-target-con-set.h index e7ba00c248..453abde6c1 100644 --- a/tcg/ppc/tcg-target-con-set.h +++ b/tcg/ppc/tcg-target-con-set.h @@ -33,7 +33,7 @@ C_O1_I2(r, r, rU) C_O1_I2(r, r, rZW) C_O1_I2(v, v, v) C_O1_I3(v, v, v, v) -C_O1_I4(v, v, v, v, v) +C_O1_I4(v, v, v, vZM, v) C_O1_I4(r, r, rC, rZ, rZ) C_O1_I4(r, r, r, ri, ri) C_O2_I1(r, r, r) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 9d07b4d8e6..3f413ce3c1 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -3725,6 +3725,11 @@ static void tcg_out_or_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2) tcg_out32(s, VOR | VRT(a0) | VRA(a1) | VRB(a2)); } +static void tcg_out_orc_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2) +{ + tcg_out32(s, VORC | VRT(a0) | VRA(a1) | VRB(a2)); +} + static void tcg_out_and_vec(TCGContext *s, TCGReg a0, TCGReg a1, TCGReg a2) { tcg_out32(s, VAND | VRT(a0) | VRA(a1) | VRB(a2)); @@ -3827,15 +3832,30 @@ static void tcg_out_cmp_vec(TCGContext *s, unsigned vece, TCGReg a0, } static void tcg_out_cmpsel_vec(TCGContext *s, unsigned vece, TCGReg a0, - TCGReg c1, TCGReg c2, TCGReg v3, TCGReg v4, - TCGCond cond) + TCGReg c1, TCGReg c2, TCGArg v3, int const_v3, + TCGReg v4, TCGCond cond) { - if (tcg_out_cmp_vec_noinv(s, vece, TCG_VEC_TMP1, c1, c2, cond)) { - TCGReg swap = v3; - v3 = v4; - v4 = swap; + bool inv = tcg_out_cmp_vec_noinv(s, vece, TCG_VEC_TMP1, c1, c2, cond); + + if (!const_v3) { + if (inv) { + tcg_out_bitsel_vec(s, a0, TCG_VEC_TMP1, v4, v3); + } else { + tcg_out_bitsel_vec(s, a0, TCG_VEC_TMP1, v3, v4); + } + } else if (v3) { + if (inv) { + tcg_out_orc_vec(s, a0, v4, TCG_VEC_TMP1); + } else { + tcg_out_or_vec(s, a0, v4, TCG_VEC_TMP1); + } + } else { + if (inv) { + tcg_out_and_vec(s, a0, v4, TCG_VEC_TMP1); + } else { + tcg_out_andc_vec(s, a0, v4, TCG_VEC_TMP1); + } } - tcg_out_bitsel_vec(s, a0, TCG_VEC_TMP1, v3, v4); } static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, @@ -3944,8 +3964,8 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_not_vec(s, a0, a1); return; case INDEX_op_orc_vec: - insn = VORC; - break; + tcg_out_orc_vec(s, a0, a1, a2); + return; case INDEX_op_nand_vec: insn = VNAND; break; @@ -3960,7 +3980,8 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_cmp_vec(s, vece, a0, a1, a2, args[3]); return; case INDEX_op_cmpsel_vec: - tcg_out_cmpsel_vec(s, vece, a0, a1, a2, args[3], args[4], args[5]); + tcg_out_cmpsel_vec(s, vece, a0, a1, a2, + args[3], const_args[3], args[4], args[5]); return; case INDEX_op_bitsel_vec: tcg_out_bitsel_vec(s, a0, a1, a2, args[3]); @@ -4330,7 +4351,7 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_ppc_msum_vec: return C_O1_I3(v, v, v, v); case INDEX_op_cmpsel_vec: - return C_O1_I4(v, v, v, v, v); + return C_O1_I4(v, v, v, vZM, v); default: g_assert_not_reached(); -- cgit v1.2.3 From 1c7d05ff70f09367ab8b519cbbb69dd5491f85f1 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 11 Sep 2024 01:54:57 +0000 Subject: tcg/s390x: Implement cmpsel_vec Do not allow cmpsel_vec to be expanded early, so that we can make the correct decision wrt the sense of the comparison. Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target-con-set.h | 1 + tcg/s390x/tcg-target.c.inc | 40 +++++++++++++++++++++------------------- tcg/s390x/tcg-target.h | 2 +- 3 files changed, 23 insertions(+), 20 deletions(-) (limited to 'tcg') diff --git a/tcg/s390x/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h index f75955eaa8..670089086d 100644 --- a/tcg/s390x/tcg-target-con-set.h +++ b/tcg/s390x/tcg-target-con-set.h @@ -38,6 +38,7 @@ C_O1_I2(r, rZ, r) C_O1_I2(v, v, r) C_O1_I2(v, v, v) C_O1_I3(v, v, v, v) +C_O1_I4(v, v, v, v, v) C_O1_I4(r, r, ri, rI, r) C_O1_I4(r, r, rC, rI, r) C_O2_I1(o, m, r) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 23935fd0f0..e044168826 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -46,6 +46,7 @@ /* A scratch register that may be be used throughout the backend. */ #define TCG_TMP0 TCG_REG_R1 +#define TCG_VEC_TMP0 TCG_REG_V31 #define TCG_GUEST_BASE_REG TCG_REG_R13 @@ -2902,6 +2903,18 @@ static void tcg_out_cmp_vec(TCGContext *s, unsigned vece, TCGReg a0, } } +static void tcg_out_cmpsel_vec(TCGContext *s, unsigned vece, TCGReg a0, + TCGReg c1, TCGReg c2, + TCGReg v3, TCGReg v4, TCGCond cond) +{ + if (tcg_out_cmp_vec_noinv(s, vece, TCG_VEC_TMP0, c1, c2, cond)) { + TCGReg swap = v3; + v3 = v4; + v4 = swap; + } + tcg_out_insn(s, VRRe, VSEL, a0, v3, v4, TCG_VEC_TMP0); +} + static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl, unsigned vece, const TCGArg args[TCG_MAX_OP_ARGS], @@ -3022,6 +3035,9 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, case INDEX_op_cmp_vec: tcg_out_cmp_vec(s, vece, a0, a1, a2, args[3]); break; + case INDEX_op_cmpsel_vec: + tcg_out_cmpsel_vec(s, vece, a0, a1, a2, args[3], args[4], args[5]); + break; case INDEX_op_s390_vuph_vec: tcg_out_insn(s, VRRa, VUPH, a0, a1, vece); @@ -3074,8 +3090,8 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) case INDEX_op_umin_vec: case INDEX_op_xor_vec: case INDEX_op_cmp_vec: - return 1; case INDEX_op_cmpsel_vec: + return 1; case INDEX_op_rotrv_vec: return -1; case INDEX_op_mul_vec: @@ -3088,17 +3104,6 @@ int tcg_can_emit_vec_op(TCGOpcode opc, TCGType type, unsigned vece) } } -static void expand_vec_cmpsel(TCGType type, unsigned vece, TCGv_vec v0, - TCGv_vec c1, TCGv_vec c2, - TCGv_vec v3, TCGv_vec v4, TCGCond cond) -{ - TCGv_vec t = tcg_temp_new_vec(type); - - tcg_gen_cmp_vec(cond, vece, t, c1, c2); - tcg_gen_bitsel_vec(vece, v0, t, v3, v4); - tcg_temp_free_vec(t); -} - static void expand_vec_sat(TCGType type, unsigned vece, TCGv_vec v0, TCGv_vec v1, TCGv_vec v2, TCGOpcode add_sub_opc) { @@ -3140,7 +3145,7 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, TCGArg a0, ...) { va_list va; - TCGv_vec v0, v1, v2, v3, v4, t0; + TCGv_vec v0, v1, v2, t0; va_start(va, a0); v0 = temp_tcgv_vec(arg_temp(a0)); @@ -3148,12 +3153,6 @@ void tcg_expand_vec_op(TCGOpcode opc, TCGType type, unsigned vece, v2 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); switch (opc) { - case INDEX_op_cmpsel_vec: - v3 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); - v4 = temp_tcgv_vec(arg_temp(va_arg(va, TCGArg))); - expand_vec_cmpsel(type, vece, v0, v1, v2, v3, v4, va_arg(va, TCGArg)); - break; - case INDEX_op_rotrv_vec: t0 = tcg_temp_new_vec(type); tcg_gen_neg_vec(vece, t0, v2); @@ -3388,6 +3387,8 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) return C_O1_I2(v, v, r); case INDEX_op_bitsel_vec: return C_O1_I3(v, v, v, v); + case INDEX_op_cmpsel_vec: + return C_O1_I4(v, v, v, v, v); default: g_assert_not_reached(); @@ -3512,6 +3513,7 @@ static void tcg_target_init(TCGContext *s) s->reserved_regs = 0; tcg_regset_set_reg(s->reserved_regs, TCG_TMP0); + tcg_regset_set_reg(s->reserved_regs, TCG_VEC_TMP0); /* XXX many insns can't be used with R0, so we better avoid it for now */ tcg_regset_set_reg(s->reserved_regs, TCG_REG_R0); tcg_regset_set_reg(s->reserved_regs, TCG_REG_CALL_STACK); diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index 62ce9d792a..86aeca166f 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -162,7 +162,7 @@ extern uint64_t s390_facilities[3]; #define TCG_TARGET_HAS_sat_vec 0 #define TCG_TARGET_HAS_minmax_vec 1 #define TCG_TARGET_HAS_bitsel_vec 1 -#define TCG_TARGET_HAS_cmpsel_vec 0 +#define TCG_TARGET_HAS_cmpsel_vec 1 #define TCG_TARGET_HAS_tst_vec 0 /* used for function call generation */ -- cgit v1.2.3 From 50695fb83e22ad011708b738d24c6c67d6296aaa Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 11 Sep 2024 03:24:10 +0000 Subject: tcg/s390x: Optimize cmpsel with constant 0/-1 arguments These can be simplified to and/or/andc/orc, avoiding the load of the constantinto a register. Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target-con-set.h | 3 ++- tcg/s390x/tcg-target-con-str.h | 1 + tcg/s390x/tcg-target.c.inc | 40 +++++++++++++++++++++++++++++++--------- 3 files changed, 34 insertions(+), 10 deletions(-) (limited to 'tcg') diff --git a/tcg/s390x/tcg-target-con-set.h b/tcg/s390x/tcg-target-con-set.h index 670089086d..370e4b1295 100644 --- a/tcg/s390x/tcg-target-con-set.h +++ b/tcg/s390x/tcg-target-con-set.h @@ -38,7 +38,8 @@ C_O1_I2(r, rZ, r) C_O1_I2(v, v, r) C_O1_I2(v, v, v) C_O1_I3(v, v, v, v) -C_O1_I4(v, v, v, v, v) +C_O1_I4(v, v, v, vZ, v) +C_O1_I4(v, v, v, vZM, v) C_O1_I4(r, r, ri, rI, r) C_O1_I4(r, r, rC, rI, r) C_O2_I1(o, m, r) diff --git a/tcg/s390x/tcg-target-con-str.h b/tcg/s390x/tcg-target-con-str.h index 745f6c0df5..3e574e0662 100644 --- a/tcg/s390x/tcg-target-con-str.h +++ b/tcg/s390x/tcg-target-con-str.h @@ -20,6 +20,7 @@ CONST('C', TCG_CT_CONST_CMP) CONST('I', TCG_CT_CONST_S16) CONST('J', TCG_CT_CONST_S32) CONST('K', TCG_CT_CONST_P32) +CONST('M', TCG_CT_CONST_M1) CONST('N', TCG_CT_CONST_INV) CONST('R', TCG_CT_CONST_INVRISBG) CONST('U', TCG_CT_CONST_U32) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index e044168826..a5d57197a4 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -36,6 +36,7 @@ #define TCG_CT_CONST_INV (1 << 13) #define TCG_CT_CONST_INVRISBG (1 << 14) #define TCG_CT_CONST_CMP (1 << 15) +#define TCG_CT_CONST_M1 (1 << 16) #define ALL_GENERAL_REGS MAKE_64BIT_MASK(0, 16) #define ALL_VECTOR_REGS MAKE_64BIT_MASK(32, 32) @@ -607,6 +608,9 @@ static bool tcg_target_const_match(int64_t val, int ct, if ((ct & TCG_CT_CONST_ZERO) && val == 0) { return true; } + if ((ct & TCG_CT_CONST_M1) && val == -1) { + return true; + } if (ct & TCG_CT_CONST_INV) { val = ~val; @@ -2904,15 +2908,30 @@ static void tcg_out_cmp_vec(TCGContext *s, unsigned vece, TCGReg a0, } static void tcg_out_cmpsel_vec(TCGContext *s, unsigned vece, TCGReg a0, - TCGReg c1, TCGReg c2, - TCGReg v3, TCGReg v4, TCGCond cond) + TCGReg c1, TCGReg c2, TCGArg v3, + int const_v3, TCGReg v4, TCGCond cond) { - if (tcg_out_cmp_vec_noinv(s, vece, TCG_VEC_TMP0, c1, c2, cond)) { - TCGReg swap = v3; - v3 = v4; - v4 = swap; + bool inv = tcg_out_cmp_vec_noinv(s, vece, TCG_VEC_TMP0, c1, c2, cond); + + if (!const_v3) { + if (inv) { + tcg_out_insn(s, VRRe, VSEL, a0, v4, v3, TCG_VEC_TMP0); + } else { + tcg_out_insn(s, VRRe, VSEL, a0, v3, v4, TCG_VEC_TMP0); + } + } else if (v3) { + if (inv) { + tcg_out_insn(s, VRRc, VOC, a0, v4, TCG_VEC_TMP0, 0); + } else { + tcg_out_insn(s, VRRc, VO, a0, v4, TCG_VEC_TMP0, 0); + } + } else { + if (inv) { + tcg_out_insn(s, VRRc, VN, a0, v4, TCG_VEC_TMP0, 0); + } else { + tcg_out_insn(s, VRRc, VNC, a0, v4, TCG_VEC_TMP0, 0); + } } - tcg_out_insn(s, VRRe, VSEL, a0, v3, v4, TCG_VEC_TMP0); } static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, @@ -3036,7 +3055,8 @@ static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, tcg_out_cmp_vec(s, vece, a0, a1, a2, args[3]); break; case INDEX_op_cmpsel_vec: - tcg_out_cmpsel_vec(s, vece, a0, a1, a2, args[3], args[4], args[5]); + tcg_out_cmpsel_vec(s, vece, a0, a1, a2, args[3], const_args[3], + args[4], args[5]); break; case INDEX_op_s390_vuph_vec: @@ -3388,7 +3408,9 @@ static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode op) case INDEX_op_bitsel_vec: return C_O1_I3(v, v, v, v); case INDEX_op_cmpsel_vec: - return C_O1_I4(v, v, v, v, v); + return (TCG_TARGET_HAS_orc_vec + ? C_O1_I4(v, v, v, vZM, v) + : C_O1_I4(v, v, v, vZ, v)); default: g_assert_not_reached(); -- cgit v1.2.3