aboutsummaryrefslogtreecommitdiff
path: root/tcg/tcg.c
diff options
context:
space:
mode:
Diffstat (limited to 'tcg/tcg.c')
-rw-r--r--tcg/tcg.c299
1 files changed, 207 insertions, 92 deletions
diff --git a/tcg/tcg.c b/tcg/tcg.c
index ca482c2301..5e53c3348f 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -60,6 +60,10 @@
#include "exec/log.h"
#include "tcg-internal.h"
+#ifdef CONFIG_TCG_INTERPRETER
+#include <ffi.h>
+#endif
+
/* Forward declarations for functions declared in tcg-target.c.inc and
used here. */
static void tcg_target_init(TCGContext *s);
@@ -143,7 +147,12 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
intptr_t arg2);
static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
TCGReg base, intptr_t ofs);
+#ifdef CONFIG_TCG_INTERPRETER
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
+ ffi_cif *cif);
+#else
static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
+#endif
static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
#ifdef TCG_TARGET_NEED_LDST_LABELS
static int tcg_out_ldst_finalize(TCGContext *s);
@@ -532,13 +541,6 @@ void tcg_pool_reset(TCGContext *s)
s->pool_current = NULL;
}
-typedef struct TCGHelperInfo {
- void *func;
- const char *name;
- unsigned flags;
- unsigned sizemask;
-} TCGHelperInfo;
-
#include "exec/helper-proto.h"
static const TCGHelperInfo all_helpers[] = {
@@ -546,6 +548,19 @@ static const TCGHelperInfo all_helpers[] = {
};
static GHashTable *helper_table;
+#ifdef CONFIG_TCG_INTERPRETER
+static GHashTable *ffi_table;
+
+static ffi_type * const typecode_to_ffi[8] = {
+ [dh_typecode_void] = &ffi_type_void,
+ [dh_typecode_i32] = &ffi_type_uint32,
+ [dh_typecode_s32] = &ffi_type_sint32,
+ [dh_typecode_i64] = &ffi_type_uint64,
+ [dh_typecode_s64] = &ffi_type_sint64,
+ [dh_typecode_ptr] = &ffi_type_pointer,
+};
+#endif
+
static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
static void process_op_defs(TCGContext *s);
static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
@@ -589,6 +604,47 @@ static void tcg_context_init(unsigned max_cpus)
(gpointer)&all_helpers[i]);
}
+#ifdef CONFIG_TCG_INTERPRETER
+ /* g_direct_hash/equal for direct comparisons on uint32_t. */
+ ffi_table = g_hash_table_new(NULL, NULL);
+ for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
+ struct {
+ ffi_cif cif;
+ ffi_type *args[];
+ } *ca;
+ uint32_t typemask = all_helpers[i].typemask;
+ gpointer hash = (gpointer)(uintptr_t)typemask;
+ ffi_status status;
+ int nargs;
+
+ if (g_hash_table_lookup(ffi_table, hash)) {
+ continue;
+ }
+
+ /* Ignoring the return type, find the last non-zero field. */
+ nargs = 32 - clz32(typemask >> 3);
+ nargs = DIV_ROUND_UP(nargs, 3);
+
+ ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
+ ca->cif.rtype = typecode_to_ffi[typemask & 7];
+ ca->cif.nargs = nargs;
+
+ if (nargs != 0) {
+ ca->cif.arg_types = ca->args;
+ for (i = 0; i < nargs; ++i) {
+ int typecode = extract32(typemask, (i + 1) * 3, 3);
+ ca->args[i] = typecode_to_ffi[typecode];
+ }
+ }
+
+ status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
+ ca->cif.rtype, ca->cif.arg_types);
+ assert(status == FFI_OK);
+
+ g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
+ }
+#endif
+
tcg_target_init(s);
process_op_defs(s);
@@ -729,10 +785,16 @@ void tcg_prologue_init(TCGContext *s)
}
#endif
- /* Assert that goto_ptr is implemented completely. */
+#ifndef CONFIG_TCG_INTERPRETER
+ /*
+ * Assert that goto_ptr is implemented completely, setting an epilogue.
+ * For tci, we use NULL as the signal to return from the interpreter,
+ * so skip this check.
+ */
if (TCG_TARGET_HAS_goto_ptr) {
tcg_debug_assert(tcg_code_gen_epilogue != NULL);
}
+#endif
}
void tcg_func_start(TCGContext *s)
@@ -1395,13 +1457,12 @@ bool tcg_op_supported(TCGOpcode op)
void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
{
int i, real_args, nb_rets, pi;
- unsigned sizemask, flags;
- TCGHelperInfo *info;
+ unsigned typemask;
+ const TCGHelperInfo *info;
TCGOp *op;
info = g_hash_table_lookup(helper_table, (gpointer)func);
- flags = info->flags;
- sizemask = info->sizemask;
+ typemask = info->typemask;
#ifdef CONFIG_PLUGIN
/* detect non-plugin helpers */
@@ -1414,36 +1475,41 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
&& !defined(CONFIG_TCG_INTERPRETER)
/* We have 64-bit values in one register, but need to pass as two
separate parameters. Split them. */
- int orig_sizemask = sizemask;
+ int orig_typemask = typemask;
int orig_nargs = nargs;
TCGv_i64 retl, reth;
TCGTemp *split_args[MAX_OPC_PARAM];
retl = NULL;
reth = NULL;
- if (sizemask != 0) {
- for (i = real_args = 0; i < nargs; ++i) {
- int is_64bit = sizemask & (1 << (i+1)*2);
- if (is_64bit) {
- TCGv_i64 orig = temp_tcgv_i64(args[i]);
- TCGv_i32 h = tcg_temp_new_i32();
- TCGv_i32 l = tcg_temp_new_i32();
- tcg_gen_extr_i64_i32(l, h, orig);
- split_args[real_args++] = tcgv_i32_temp(h);
- split_args[real_args++] = tcgv_i32_temp(l);
- } else {
- split_args[real_args++] = args[i];
- }
+ typemask = 0;
+ for (i = real_args = 0; i < nargs; ++i) {
+ int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
+ bool is_64bit = (argtype & ~1) == dh_typecode_i64;
+
+ if (is_64bit) {
+ TCGv_i64 orig = temp_tcgv_i64(args[i]);
+ TCGv_i32 h = tcg_temp_new_i32();
+ TCGv_i32 l = tcg_temp_new_i32();
+ tcg_gen_extr_i64_i32(l, h, orig);
+ split_args[real_args++] = tcgv_i32_temp(h);
+ typemask |= dh_typecode_i32 << (real_args * 3);
+ split_args[real_args++] = tcgv_i32_temp(l);
+ typemask |= dh_typecode_i32 << (real_args * 3);
+ } else {
+ split_args[real_args++] = args[i];
+ typemask |= argtype << (real_args * 3);
}
- nargs = real_args;
- args = split_args;
- sizemask = 0;
}
+ nargs = real_args;
+ args = split_args;
#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
for (i = 0; i < nargs; ++i) {
- int is_64bit = sizemask & (1 << (i+1)*2);
- int is_signed = sizemask & (2 << (i+1)*2);
- if (!is_64bit) {
+ int argtype = extract32(typemask, (i + 1) * 3, 3);
+ bool is_32bit = (argtype & ~1) == dh_typecode_i32;
+ bool is_signed = argtype & 1;
+
+ if (is_32bit) {
TCGv_i64 temp = tcg_temp_new_i64();
TCGv_i64 orig = temp_tcgv_i64(args[i]);
if (is_signed) {
@@ -1462,7 +1528,7 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
if (ret != NULL) {
#if defined(__sparc__) && !defined(__arch64__) \
&& !defined(CONFIG_TCG_INTERPRETER)
- if (orig_sizemask & 1) {
+ if ((typemask & 6) == dh_typecode_i64) {
/* The 32-bit ABI is going to return the 64-bit value in
the %o0/%o1 register pair. Prepare for this by using
two return temporaries, and reassemble below. */
@@ -1476,7 +1542,7 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
nb_rets = 1;
}
#else
- if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) {
+ if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
#ifdef HOST_WORDS_BIGENDIAN
op->args[pi++] = temp_arg(ret + 1);
op->args[pi++] = temp_arg(ret);
@@ -1497,25 +1563,39 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
real_args = 0;
for (i = 0; i < nargs; i++) {
- int is_64bit = sizemask & (1 << (i+1)*2);
- if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
-#ifdef TCG_TARGET_CALL_ALIGN_ARGS
- /* some targets want aligned 64 bit args */
- if (real_args & 1) {
- op->args[pi++] = TCG_CALL_DUMMY_ARG;
- real_args++;
- }
+ int argtype = extract32(typemask, (i + 1) * 3, 3);
+ bool is_64bit = (argtype & ~1) == dh_typecode_i64;
+ bool want_align = false;
+
+#if defined(CONFIG_TCG_INTERPRETER)
+ /*
+ * Align all arguments, so that they land in predictable places
+ * for passing off to ffi_call.
+ */
+ want_align = true;
+#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
+ /* Some targets want aligned 64 bit args */
+ want_align = is_64bit;
#endif
- /* If stack grows up, then we will be placing successive
- arguments at lower addresses, which means we need to
- reverse the order compared to how we would normally
- treat either big or little-endian. For those arguments
- that will wind up in registers, this still works for
- HPPA (the only current STACK_GROWSUP target) since the
- argument registers are *also* allocated in decreasing
- order. If another such target is added, this logic may
- have to get more complicated to differentiate between
- stack arguments and register arguments. */
+
+ if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
+ op->args[pi++] = TCG_CALL_DUMMY_ARG;
+ real_args++;
+ }
+
+ if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
+ /*
+ * If stack grows up, then we will be placing successive
+ * arguments at lower addresses, which means we need to
+ * reverse the order compared to how we would normally
+ * treat either big or little-endian. For those arguments
+ * that will wind up in registers, this still works for
+ * HPPA (the only current STACK_GROWSUP target) since the
+ * argument registers are *also* allocated in decreasing
+ * order. If another such target is added, this logic may
+ * have to get more complicated to differentiate between
+ * stack arguments and register arguments.
+ */
#if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
op->args[pi++] = temp_arg(args[i] + 1);
op->args[pi++] = temp_arg(args[i]);
@@ -1531,7 +1611,7 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
real_args++;
}
op->args[pi++] = (uintptr_t)func;
- op->args[pi++] = flags;
+ op->args[pi++] = (uintptr_t)info;
TCGOP_CALLI(op) = real_args;
/* Make sure the fields didn't overflow. */
@@ -1542,7 +1622,9 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
&& !defined(CONFIG_TCG_INTERPRETER)
/* Free all of the parts we allocated above. */
for (i = real_args = 0; i < orig_nargs; ++i) {
- int is_64bit = orig_sizemask & (1 << (i+1)*2);
+ int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
+ bool is_64bit = (argtype & ~1) == dh_typecode_i64;
+
if (is_64bit) {
tcg_temp_free_internal(args[real_args++]);
tcg_temp_free_internal(args[real_args++]);
@@ -1550,7 +1632,7 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
real_args++;
}
}
- if (orig_sizemask & 1) {
+ if ((orig_typemask & 6) == dh_typecode_i64) {
/* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
Note that describing these as TCGv_i64 eliminates an unnecessary
zero-extension that tcg_gen_concat_i32_i64 would create. */
@@ -1560,8 +1642,10 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
}
#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
for (i = 0; i < nargs; ++i) {
- int is_64bit = sizemask & (1 << (i+1)*2);
- if (!is_64bit) {
+ int argtype = extract32(typemask, (i + 1) * 3, 3);
+ bool is_32bit = (argtype & ~1) == dh_typecode_i32;
+
+ if (is_32bit) {
tcg_temp_free_internal(args[i]);
}
}
@@ -1646,19 +1730,6 @@ static char *tcg_get_arg_str(TCGContext *s, char *buf,
return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
}
-/* Find helper name. */
-static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)
-{
- const char *ret = NULL;
- if (helper_table) {
- TCGHelperInfo *info = g_hash_table_lookup(helper_table, (gpointer)val);
- if (info) {
- ret = info->name;
- }
- }
- return ret;
-}
-
static const char * const cond_name[] =
{
[TCG_COND_NEVER] = "never",
@@ -1749,15 +1820,28 @@ static void tcg_dump_ops(TCGContext *s, bool have_prefs)
col += qemu_log(" " TARGET_FMT_lx, a);
}
} else if (c == INDEX_op_call) {
+ const TCGHelperInfo *info = tcg_call_info(op);
+ void *func = tcg_call_func(op);
+
/* variable number of arguments */
nb_oargs = TCGOP_CALLO(op);
nb_iargs = TCGOP_CALLI(op);
nb_cargs = def->nb_cargs;
- /* function name, flags, out args */
- col += qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name,
- tcg_find_helper(s, op->args[nb_oargs + nb_iargs]),
- op->args[nb_oargs + nb_iargs + 1], nb_oargs);
+ col += qemu_log(" %s ", def->name);
+
+ /*
+ * Print the function name from TCGHelperInfo, if available.
+ * Note that plugins have a template function for the info,
+ * but the actual function pointer comes from the plugin.
+ */
+ if (func == info->func) {
+ col += qemu_log("%s", info->name);
+ } else {
+ col += qemu_log("plugin(%p)", func);
+ }
+
+ col += qemu_log("$0x%x,$%d", info->flags, nb_oargs);
for (i = 0; i < nb_oargs; i++) {
col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
op->args[i]));
@@ -2144,7 +2228,6 @@ static void reachable_code_pass(TCGContext *s)
QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
bool remove = dead;
TCGLabel *label;
- int call_flags;
switch (op->opc) {
case INDEX_op_set_label:
@@ -2189,8 +2272,7 @@ static void reachable_code_pass(TCGContext *s)
case INDEX_op_call:
/* Notice noreturn helper calls, raising exceptions. */
- call_flags = op->args[TCGOP_CALLO(op) + TCGOP_CALLI(op) + 1];
- if (call_flags & TCG_CALL_NO_RETURN) {
+ if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
dead = true;
}
break;
@@ -2391,7 +2473,7 @@ static void liveness_pass_1(TCGContext *s)
nb_oargs = TCGOP_CALLO(op);
nb_iargs = TCGOP_CALLI(op);
- call_flags = op->args[nb_oargs + nb_iargs + 1];
+ call_flags = tcg_call_flags(op);
/* pure functions can be removed if their result is unused */
if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
@@ -2706,7 +2788,7 @@ static bool liveness_pass_2(TCGContext *s)
if (opc == INDEX_op_call) {
nb_oargs = TCGOP_CALLO(op);
nb_iargs = TCGOP_CALLI(op);
- call_flags = op->args[nb_oargs + nb_iargs + 1];
+ call_flags = tcg_call_flags(op);
} else {
nb_iargs = def->nb_iargs;
nb_oargs = def->nb_oargs;
@@ -2933,20 +3015,42 @@ static void check_regs(TCGContext *s)
static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
{
-#if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
- /* Sparc64 stack is accessed with offset of 2047 */
- s->current_frame_offset = (s->current_frame_offset +
- (tcg_target_long)sizeof(tcg_target_long) - 1) &
- ~(sizeof(tcg_target_long) - 1);
-#endif
- if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) >
- s->frame_end) {
- tcg_abort();
+ intptr_t off, size, align;
+
+ switch (ts->type) {
+ case TCG_TYPE_I32:
+ size = align = 4;
+ break;
+ case TCG_TYPE_I64:
+ case TCG_TYPE_V64:
+ size = align = 8;
+ break;
+ case TCG_TYPE_V128:
+ size = align = 16;
+ break;
+ case TCG_TYPE_V256:
+ /* Note that we do not require aligned storage for V256. */
+ size = 32, align = 16;
+ break;
+ default:
+ g_assert_not_reached();
}
- ts->mem_offset = s->current_frame_offset;
+
+ assert(align <= TCG_TARGET_STACK_ALIGN);
+ off = ROUND_UP(s->current_frame_offset, align);
+
+ /* If we've exhausted the stack frame, restart with a smaller TB. */
+ if (off + size > s->frame_end) {
+ tcg_raise_tb_overflow(s);
+ }
+ s->current_frame_offset = off + size;
+
+ ts->mem_offset = off;
+#if defined(__sparc__)
+ ts->mem_offset += TCG_TARGET_STACK_BIAS;
+#endif
ts->mem_base = s->frame_temp;
ts->mem_allocated = 1;
- s->current_frame_offset += sizeof(tcg_target_long);
}
static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
@@ -3777,6 +3881,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
const int nb_oargs = TCGOP_CALLO(op);
const int nb_iargs = TCGOP_CALLI(op);
const TCGLifeData arg_life = op->life;
+ const TCGHelperInfo *info;
int flags, nb_regs, i;
TCGReg reg;
TCGArg arg;
@@ -3787,8 +3892,9 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
int allocate_args;
TCGRegSet allocated_regs;
- func_addr = (tcg_insn_unit *)(intptr_t)op->args[nb_oargs + nb_iargs];
- flags = op->args[nb_oargs + nb_iargs + 1];
+ func_addr = tcg_call_func(op);
+ info = tcg_call_info(op);
+ flags = info->flags;
nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
if (nb_regs > nb_iargs) {
@@ -3880,7 +3986,16 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
save_globals(s, allocated_regs);
}
+#ifdef CONFIG_TCG_INTERPRETER
+ {
+ gpointer hash = (gpointer)(uintptr_t)info->typemask;
+ ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
+ assert(cif != NULL);
+ tcg_out_call(s, func_addr, cif);
+ }
+#else
tcg_out_call(s, func_addr);
+#endif
/* assign output registers and emit moves if needed */
for(i = 0; i < nb_oargs; i++) {
generated by cgit v1.2.3 (git 2.26.2) at 2025-03-21 06:21:19 +0000