tcg: Add tcg-be-ldst.h

Move TCGLabelQemuLdst and related stuff out of tcg.h.

Signed-off-by: Richard Henderson <rth@twiddle.net>

8 files changed, 112 insertions, 158 deletions

diff --git a/tcg/aarch64/tcg-target.c b/tcg/aarch64/tcg-target.c
index 78e1443808..04d7ae328d 100644
--- a/tcg/aarch64/tcg-target.c
+++ b/tcg/aarch64/tcg-target.c
@@ -10,6 +10,7 @@
  * See the COPYING file in the top-level directory for details.
  */
 
+#include "tcg-be-ldst.h"
 #include "qemu/bitops.h"
 
 #ifndef NDEBUG
@@ -834,33 +835,13 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
     tcg_out_goto(s, (tcg_target_long)lb->raddr);
 }
 
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i;
-    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
-        TCGLabelQemuLdst *label = &s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path(s, label);
-        } else {
-            tcg_out_qemu_st_slow_path(s, label);
-        }
-    }
-}
-
 static void add_qemu_ldst_label(TCGContext *s, int is_ld, int opc,
                                 TCGReg data_reg, TCGReg addr_reg,
                                 int mem_index,
                                 uint8_t *raddr, uint8_t *label_ptr)
 {
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
+    TCGLabelQemuLdst *label = new_ldst_label(s);
 
-    idx = s->nb_qemu_ldst_labels++;
-    label = &s->qemu_ldst_labels[idx];
     label->is_ld = is_ld;
     label->opc = opc;
     label->datalo_reg = data_reg;
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
index 622cc49aa7..c0e14661b2 100644
--- a/tcg/arm/tcg-target.c
+++ b/tcg/arm/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-ldst.h"
+
 /* The __ARM_ARCH define is provided by gcc 4.8.  Construct it otherwise.  */
 #ifndef __ARM_ARCH
 # if defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) \
@@ -1243,15 +1245,8 @@ static void add_qemu_ldst_label(TCGContext *s, int is_ld, int opc,
                                 int addrhi_reg, int mem_index,
                                 uint8_t *raddr, uint8_t *label_ptr)
 {
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
+    TCGLabelQemuLdst *label = new_ldst_label(s);
 
-    idx = s->nb_qemu_ldst_labels++;
-    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
     label->is_ld = is_ld;
     label->opc = opc;
     label->datalo_reg = data_reg;
@@ -1968,22 +1963,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     }
 }
 
-#ifdef CONFIG_SOFTMMU
-/* Generate TB finalization at the end of block.  */
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i;
-    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
-        TCGLabelQemuLdst *label = &s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path(s, label);
-        } else {
-            tcg_out_qemu_st_slow_path(s, label);
-        }
-    }
-}
-#endif /* SOFTMMU */
-
 static const TCGTargetOpDef arm_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
index c1f07415ab..b865b4b662 100644
--- a/tcg/i386/tcg-target.c
+++ b/tcg/i386/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-ldst.h"
+
 #ifndef NDEBUG
 static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #if TCG_TARGET_REG_BITS == 64
@@ -1455,15 +1457,8 @@ static void add_qemu_ldst_label(TCGContext *s,
                                 uint8_t *raddr,
                                 uint8_t **label_ptr)
 {
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
+    TCGLabelQemuLdst *label = new_ldst_label(s);
 
-    idx = s->nb_qemu_ldst_labels++;
-    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
     label->is_ld = is_ld;
     label->opc = opc;
     label->datalo_reg = data_reg;
@@ -1628,25 +1623,6 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     tcg_out_push(s, retaddr);
     tcg_out_jmp(s, (uintptr_t)qemu_st_helpers[s_bits]);
 }
-
-/*
- * Generate TB finalization at the end of block
- */
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i;
-    TCGLabelQemuLdst *label;
-
-    /* qemu_ld/st slow paths */
-    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
-        label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path(s, label);
-        } else {
-            tcg_out_qemu_st_slow_path(s, label);
-        }
-    }
-}
 #endif  /* CONFIG_SOFTMMU */
 
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
diff --git a/tcg/ppc/tcg-target.c b/tcg/ppc/tcg-target.c
index 97e33edcfd..68778c2bc4 100644
--- a/tcg/ppc/tcg-target.c
+++ b/tcg/ppc/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-ldst.h"
+
 static uint8_t *tb_ret_addr;
 
 #if defined _CALL_DARWIN || defined __APPLE__
@@ -532,15 +534,8 @@ static void add_qemu_ldst_label (TCGContext *s,
                                  uint8_t *raddr,
                                  uint8_t *label_ptr)
 {
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
+    TCGLabelQemuLdst *label = new_ldst_label(s);
 
-    idx = s->nb_qemu_ldst_labels++;
-    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
     label->is_ld = is_ld;
     label->opc = opc;
     label->datalo_reg = data_reg;
@@ -889,23 +884,6 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l)
     tcg_out_b(s, LK, (uintptr_t)st_trampolines[l->opc]);
     tcg_out_b(s, 0, (uintptr_t)l->raddr);
 }
-
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i;
-    TCGLabelQemuLdst *label;
-
-    /* qemu_ld/st slow paths */
-    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
-        label = (TCGLabelQemuLdst *) &s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path (s, label);
-        }
-        else {
-            tcg_out_qemu_st_slow_path (s, label);
-        }
-    }
-}
 #endif
 
 #ifdef CONFIG_SOFTMMU
diff --git a/tcg/ppc64/tcg-target.c b/tcg/ppc64/tcg-target.c
index 332f4d8df1..12c1f61b03 100644
--- a/tcg/ppc64/tcg-target.c
+++ b/tcg/ppc64/tcg-target.c
@@ -22,6 +22,8 @@
  * THE SOFTWARE.
  */
 
+#include "tcg-be-ldst.h"
+
 #define TCG_CT_CONST_S16  0x100
 #define TCG_CT_CONST_U16  0x200
 #define TCG_CT_CONST_S32  0x400
@@ -931,15 +933,8 @@ static void add_qemu_ldst_label(TCGContext *s, bool is_ld, int opc,
                                 int data_reg, int addr_reg, int mem_index,
                                 uint8_t *raddr, uint8_t *label_ptr)
 {
-    int idx;
-    TCGLabelQemuLdst *label;
-
-    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
-        tcg_abort();
-    }
+    TCGLabelQemuLdst *label = new_ldst_label(s);
 
-    idx = s->nb_qemu_ldst_labels++;
-    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
     label->is_ld = is_ld;
     label->opc = opc;
     label->datalo_reg = data_reg;
@@ -998,21 +993,6 @@ static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
 
     tcg_out_b(s, 0, (uintptr_t)lb->raddr);
 }
-
-void tcg_out_tb_finalize(TCGContext *s)
-{
-    int i, n = s->nb_qemu_ldst_labels;
-
-    /* qemu_ld/st slow paths */
-    for (i = 0; i < n; i++) {
-        TCGLabelQemuLdst *label = &s->qemu_ldst_labels[i];
-        if (label->is_ld) {
-            tcg_out_qemu_ld_slow_path(s, label);
-        } else {
-            tcg_out_qemu_st_slow_path(s, label);
-        }
-    }
-}
 #endif /* SOFTMMU */
 
 static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
diff --git a/tcg/tcg-be-ldst.h b/tcg/tcg-be-ldst.h
new file mode 100644
index 0000000000..2826d296d7
--- /dev/null
+++ b/tcg/tcg-be-ldst.h
@@ -0,0 +1,90 @@
+/*
+ * TCG Backend Data: load-store optimization only.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#ifdef CONFIG_SOFTMMU
+#define TCG_MAX_QEMU_LDST       640
+
+typedef struct TCGLabelQemuLdst {
+    int is_ld:1;            /* qemu_ld: 1, qemu_st: 0 */
+    int opc:4;
+    TCGReg addrlo_reg;      /* reg index for low word of guest virtual addr */
+    TCGReg addrhi_reg;      /* reg index for high word of guest virtual addr */
+    TCGReg datalo_reg;      /* reg index for low word to be loaded or stored */
+    TCGReg datahi_reg;      /* reg index for high word to be loaded or stored */
+    int mem_index;          /* soft MMU memory index */
+    uint8_t *raddr;         /* gen code addr of the next IR of qemu_ld/st IR */
+    uint8_t *label_ptr[2];  /* label pointers to be updated */
+} TCGLabelQemuLdst;
+
+typedef struct TCGBackendData {
+    int nb_ldst_labels;
+    TCGLabelQemuLdst ldst_labels[TCG_MAX_QEMU_LDST];
+} TCGBackendData;
+
+
+/*
+ * Initialize TB backend data at the beginning of the TB.
+ */
+
+static inline void tcg_out_tb_init(TCGContext *s)
+{
+    s->be->nb_ldst_labels = 0;
+}
+
+/*
+ * Generate TB finalization at the end of block
+ */
+
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l);
+
+static void tcg_out_tb_finalize(TCGContext *s)
+{
+    TCGLabelQemuLdst *lb = s->be->ldst_labels;
+    int i, n = s->be->nb_ldst_labels;
+
+    /* qemu_ld/st slow paths */
+    for (i = 0; i < n; i++) {
+        if (lb[i].is_ld) {
+            tcg_out_qemu_ld_slow_path(s, lb + i);
+        } else {
+            tcg_out_qemu_st_slow_path(s, lb + i);
+        }
+    }
+}
+
+/*
+ * Allocate a new TCGLabelQemuLdst entry.
+ */
+
+static inline TCGLabelQemuLdst *new_ldst_label(TCGContext *s)
+{
+    TCGBackendData *be = s->be;
+    int n = be->nb_ldst_labels;
+
+    assert(n < TCG_MAX_QEMU_LDST);
+    be->nb_ldst_labels = n + 1;
+    return &be->ldst_labels[n];
+}
+#else
+#include "tcg-be-null.h"
+#endif /* CONFIG_SOFTMMU */
diff --git a/tcg/tcg.c b/tcg/tcg.c
index 3b88c56340..eba080a7e0 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -103,6 +103,9 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
                        intptr_t arg2);
 static int tcg_target_const_match(tcg_target_long val,
                                   const TCGArgConstraint *arg_ct);
+static void tcg_out_tb_init(TCGContext *s);
+static void tcg_out_tb_finalize(TCGContext *s);
+
 
 TCGOpDef tcg_op_defs[] = {
 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
@@ -370,13 +373,7 @@ void tcg_func_start(TCGContext *s)
     s->gen_opc_ptr = s->gen_opc_buf;
     s->gen_opparam_ptr = s->gen_opparam_buf;
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-    /* Initialize qemu_ld/st labels to assist code generation at the end of TB
-       for TLB miss cases at the end of TB */
-    s->qemu_ldst_labels = tcg_malloc(sizeof(TCGLabelQemuLdst) *
-                                     TCG_MAX_QEMU_LDST);
-    s->nb_qemu_ldst_labels = 0;
-#endif
+    s->be = tcg_malloc(sizeof(TCGBackendData));
 }
 
 static inline void tcg_temp_alloc(TCGContext *s, int n)
@@ -2297,6 +2294,8 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
     s->code_buf = gen_code_buf;
     s->code_ptr = gen_code_buf;
 
+    tcg_out_tb_init(s);
+
     args = s->gen_opparam_buf;
     op_index = 0;
 
@@ -2370,10 +2369,8 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
 #endif
     }
  the_end:
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
     /* Generate TB finalization at the end of block */
     tcg_out_tb_finalize(s);
-#endif
     return -1;
 }
 
diff --git a/tcg/tcg.h b/tcg/tcg.h
index f67fdb61a1..7d0c6523f2 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -211,24 +211,6 @@ typedef tcg_target_ulong TCGArg;
    are aliases for target_ulong and host pointer sized values respectively.
  */
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-/* Macros/structures for qemu_ld/st IR code optimization:
-   TCG_MAX_HELPER_LABELS is defined as same as OPC_BUF_SIZE in exec-all.h. */
-#define TCG_MAX_QEMU_LDST       640
-
-typedef struct TCGLabelQemuLdst {
-    int is_ld:1;            /* qemu_ld: 1, qemu_st: 0 */
-    int opc:4;
-    int addrlo_reg;         /* reg index for low word of guest virtual addr */
-    int addrhi_reg;         /* reg index for high word of guest virtual addr */
-    int datalo_reg;         /* reg index for low word to be loaded or stored */
-    int datahi_reg;         /* reg index for high word to be loaded or stored */
-    int mem_index;          /* soft MMU memory index */
-    uint8_t *raddr;         /* gen code addr of the next IR of qemu_ld/st IR */
-    uint8_t *label_ptr[2];  /* label pointers to be updated */
-} TCGLabelQemuLdst;
-#endif
-
 #ifdef CONFIG_DEBUG_TCG
 #define DEBUG_TCGV 1
 #endif
@@ -488,12 +470,8 @@ struct TCGContext {
 
     TBContext tb_ctx;
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-    /* labels info for qemu_ld/st IRs
-       The labels help to generate TLB miss case codes at the end of TB */
-    TCGLabelQemuLdst *qemu_ldst_labels;
-    int nb_qemu_ldst_labels;
-#endif
+    /* The TCGBackendData structure is private to tcg-target.c.  */
+    struct TCGBackendData *be;
 };
 
 extern TCGContext tcg_ctx;
@@ -735,11 +713,6 @@ TCGv_i64 tcg_const_local_i64(int64_t val);
 
 void tcg_register_jit(void *buf, size_t buf_size);
 
-#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
-/* Generate TB finalization at the end of block */
-void tcg_out_tb_finalize(TCGContext *s);
-#endif
-
 /*
  * Memory helpers that will be used by TCG generated code.
  */