aboutsummaryrefslogtreecommitdiff
path: root/accel
diff options
context:
space:
mode:
Diffstat (limited to 'accel')
-rw-r--r--accel/tcg/cpu-exec.c24
-rw-r--r--accel/tcg/internal.h91
-rw-r--r--accel/tcg/meson.build1
-rw-r--r--accel/tcg/tb-maint.c704
-rw-r--r--accel/tcg/tcg-accel-ops-mttcg.c5
-rw-r--r--accel/tcg/tcg-accel-ops-rr.c7
-rw-r--r--accel/tcg/translate-all.c1017
-rw-r--r--accel/tcg/translator.c9
-rw-r--r--accel/tcg/user-exec.c42
9 files changed, 915 insertions, 985 deletions
diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index f9e5cc9ba0..82b06c1824 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -187,13 +187,14 @@ static bool tb_lookup_cmp(const void *p, const void *d)
const struct tb_desc *desc = d;
if ((TARGET_TB_PCREL || tb_pc(tb) == desc->pc) &&
- tb->page_addr[0] == desc->page_addr0 &&
+ tb_page_addr0(tb) == desc->page_addr0 &&
tb->cs_base == desc->cs_base &&
tb->flags == desc->flags &&
tb->trace_vcpu_dstate == desc->trace_vcpu_dstate &&
tb_cflags(tb) == desc->cflags) {
/* check next page if needed */
- if (tb->page_addr[1] == -1) {
+ tb_page_addr_t tb_phys_page1 = tb_page_addr1(tb);
+ if (tb_phys_page1 == -1) {
return true;
} else {
tb_page_addr_t phys_page1;
@@ -210,7 +211,7 @@ static bool tb_lookup_cmp(const void *p, const void *d)
*/
virt_page1 = TARGET_PAGE_ALIGN(desc->pc);
phys_page1 = get_page_addr_code(desc->env, virt_page1);
- if (tb->page_addr[1] == phys_page1) {
+ if (tb_phys_page1 == phys_page1) {
return true;
}
}
@@ -304,16 +305,12 @@ static void log_cpu_exec(target_ulong pc, CPUState *cpu,
}
}
-static bool check_for_breakpoints(CPUState *cpu, target_ulong pc,
- uint32_t *cflags)
+static bool check_for_breakpoints_slow(CPUState *cpu, target_ulong pc,
+ uint32_t *cflags)
{
CPUBreakpoint *bp;
bool match_page = false;
- if (likely(QTAILQ_EMPTY(&cpu->breakpoints))) {
- return false;
- }
-
/*
* Singlestep overrides breakpoints.
* This requirement is visible in the record-replay tests, where
@@ -374,6 +371,13 @@ static bool check_for_breakpoints(CPUState *cpu, target_ulong pc,
return false;
}
+static inline bool check_for_breakpoints(CPUState *cpu, target_ulong pc,
+ uint32_t *cflags)
+{
+ return unlikely(!QTAILQ_EMPTY(&cpu->breakpoints)) &&
+ check_for_breakpoints_slow(cpu, pc, cflags);
+}
+
/**
* helper_lookup_tb_ptr: quick check for next tb
* @env: current cpu state
@@ -1016,7 +1020,7 @@ int cpu_exec(CPUState *cpu)
* direct jump to a TB spanning two pages because the mapping
* for the second page can change.
*/
- if (tb->page_addr[1] != -1) {
+ if (tb_page_addr1(tb) != -1) {
last_tb = NULL;
}
#endif
diff --git a/accel/tcg/internal.h b/accel/tcg/internal.h
index dc800fd485..1227bb69bd 100644
--- a/accel/tcg/internal.h
+++ b/accel/tcg/internal.h
@@ -11,12 +11,103 @@
#include "exec/exec-all.h"
+/*
+ * Access to the various translations structures need to be serialised
+ * via locks for consistency. In user-mode emulation access to the
+ * memory related structures are protected with mmap_lock.
+ * In !user-mode we use per-page locks.
+ */
+#ifdef CONFIG_SOFTMMU
+#define assert_memory_lock()
+#else
+#define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
+#endif
+
+typedef struct PageDesc {
+ /* list of TBs intersecting this ram page */
+ uintptr_t first_tb;
+#ifdef CONFIG_USER_ONLY
+ unsigned long flags;
+ void *target_data;
+#endif
+#ifdef CONFIG_SOFTMMU
+ QemuSpin lock;
+#endif
+} PageDesc;
+
+/* Size of the L2 (and L3, etc) page tables. */
+#define V_L2_BITS 10
+#define V_L2_SIZE (1 << V_L2_BITS)
+
+/*
+ * L1 Mapping properties
+ */
+extern int v_l1_size;
+extern int v_l1_shift;
+extern int v_l2_levels;
+
+/*
+ * The bottom level has pointers to PageDesc, and is indexed by
+ * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
+ */
+#define V_L1_MIN_BITS 4
+#define V_L1_MAX_BITS (V_L2_BITS + 3)
+#define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
+
+extern void *l1_map[V_L1_MAX_SIZE];
+
+PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc);
+
+static inline PageDesc *page_find(tb_page_addr_t index)
+{
+ return page_find_alloc(index, false);
+}
+
+/* list iterators for lists of tagged pointers in TranslationBlock */
+#define TB_FOR_EACH_TAGGED(head, tb, n, field) \
+ for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \
+ tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
+ tb = (TranslationBlock *)((uintptr_t)tb & ~1))
+
+#define PAGE_FOR_EACH_TB(pagedesc, tb, n) \
+ TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
+
+#define TB_FOR_EACH_JMP(head_tb, tb, n) \
+ TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
+
+/* In user-mode page locks aren't used; mmap_lock is enough */
+#ifdef CONFIG_USER_ONLY
+#define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
+static inline void page_lock(PageDesc *pd) { }
+static inline void page_unlock(PageDesc *pd) { }
+#else
+#ifdef CONFIG_DEBUG_TCG
+void do_assert_page_locked(const PageDesc *pd, const char *file, int line);
+#define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
+#else
+#define assert_page_locked(pd)
+#endif
+void page_lock(PageDesc *pd);
+void page_unlock(PageDesc *pd);
+#endif
+#if !defined(CONFIG_USER_ONLY) && defined(CONFIG_DEBUG_TCG)
+void assert_no_pages_locked(void);
+#else
+static inline void assert_no_pages_locked(void) { }
+#endif
+
TranslationBlock *tb_gen_code(CPUState *cpu, target_ulong pc,
target_ulong cs_base, uint32_t flags,
int cflags);
G_NORETURN void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr);
void page_init(void);
void tb_htable_init(void);
+void tb_reset_jump(TranslationBlock *tb, int n);
+TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+ tb_page_addr_t phys_page2);
+bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc);
+int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
+ uintptr_t searched_pc, bool reset_icount);
/* Return the current PC from CPU, which may be cached in TB. */
static inline target_ulong log_pc(CPUState *cpu, const TranslationBlock *tb)
diff --git a/accel/tcg/meson.build b/accel/tcg/meson.build
index 7a0a79d731..75e1dffb4d 100644
--- a/accel/tcg/meson.build
+++ b/accel/tcg/meson.build
@@ -3,6 +3,7 @@ tcg_ss.add(files(
'tcg-all.c',
'cpu-exec-common.c',
'cpu-exec.c',
+ 'tb-maint.c',
'tcg-runtime-gvec.c',
'tcg-runtime.c',
'translate-all.c',
diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c
new file mode 100644
index 0000000000..c8e921089d
--- /dev/null
+++ b/accel/tcg/tb-maint.c
@@ -0,0 +1,704 @@
+/*
+ * Translation Block Maintaince
+ *
+ * Copyright (c) 2003 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "qemu/osdep.h"
+#include "exec/cputlb.h"
+#include "exec/log.h"
+#include "exec/exec-all.h"
+#include "exec/translate-all.h"
+#include "sysemu/tcg.h"
+#include "tcg/tcg.h"
+#include "tb-hash.h"
+#include "tb-context.h"
+#include "internal.h"
+
+
+static bool tb_cmp(const void *ap, const void *bp)
+{
+ const TranslationBlock *a = ap;
+ const TranslationBlock *b = bp;
+
+ return ((TARGET_TB_PCREL || tb_pc(a) == tb_pc(b)) &&
+ a->cs_base == b->cs_base &&
+ a->flags == b->flags &&
+ (tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) &&
+ a->trace_vcpu_dstate == b->trace_vcpu_dstate &&
+ tb_page_addr0(a) == tb_page_addr0(b) &&
+ tb_page_addr1(a) == tb_page_addr1(b));
+}
+
+void tb_htable_init(void)
+{
+ unsigned int mode = QHT_MODE_AUTO_RESIZE;
+
+ qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
+}
+
+/* Set to NULL all the 'first_tb' fields in all PageDescs. */
+static void page_flush_tb_1(int level, void **lp)
+{
+ int i;
+
+ if (*lp == NULL) {
+ return;
+ }
+ if (level == 0) {
+ PageDesc *pd = *lp;
+
+ for (i = 0; i < V_L2_SIZE; ++i) {
+ page_lock(&pd[i]);
+ pd[i].first_tb = (uintptr_t)NULL;
+ page_unlock(&pd[i]);
+ }
+ } else {
+ void **pp = *lp;
+
+ for (i = 0; i < V_L2_SIZE; ++i) {
+ page_flush_tb_1(level - 1, pp + i);
+ }
+ }
+}
+
+static void page_flush_tb(void)
+{
+ int i, l1_sz = v_l1_size;
+
+ for (i = 0; i < l1_sz; i++) {
+ page_flush_tb_1(v_l2_levels, l1_map + i);
+ }
+}
+
+/* flush all the translation blocks */
+static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
+{
+ bool did_flush = false;
+
+ mmap_lock();
+ /* If it is already been done on request of another CPU, just retry. */
+ if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
+ goto done;
+ }
+ did_flush = true;
+
+ CPU_FOREACH(cpu) {
+ tcg_flush_jmp_cache(cpu);
+ }
+
+ qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
+ page_flush_tb();
+
+ tcg_region_reset_all();
+ /* XXX: flush processor icache at this point if cache flush is expensive */
+ qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
+
+done:
+ mmap_unlock();
+ if (did_flush) {
+ qemu_plugin_flush_cb();
+ }
+}
+
+void tb_flush(CPUState *cpu)
+{
+ if (tcg_enabled()) {
+ unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count);
+
+ if (cpu_in_exclusive_context(cpu)) {
+ do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
+ } else {
+ async_safe_run_on_cpu(cpu, do_tb_flush,
+ RUN_ON_CPU_HOST_INT(tb_flush_count));
+ }
+ }
+}
+
+/*
+ * user-mode: call with mmap_lock held
+ * !user-mode: call with @pd->lock held
+ */
+static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb)
+{
+ TranslationBlock *tb1;
+ uintptr_t *pprev;
+ unsigned int n1;
+
+ assert_page_locked(pd);
+ pprev = &pd->first_tb;
+ PAGE_FOR_EACH_TB(pd, tb1, n1) {
+ if (tb1 == tb) {
+ *pprev = tb1->page_next[n1];
+ return;
+ }
+ pprev = &tb1->page_next[n1];
+ }
+ g_assert_not_reached();
+}
+
+/* remove @orig from its @n_orig-th jump list */
+static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
+{
+ uintptr_t ptr, ptr_locked;
+ TranslationBlock *dest;
+ TranslationBlock *tb;
+ uintptr_t *pprev;
+ int n;
+
+ /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
+ ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1);
+ dest = (TranslationBlock *)(ptr & ~1);
+ if (dest == NULL) {
+ return;
+ }
+
+ qemu_spin_lock(&dest->jmp_lock);
+ /*
+ * While acquiring the lock, the jump might have been removed if the
+ * destination TB was invalidated; check again.
+ */
+ ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]);
+ if (ptr_locked != ptr) {
+ qemu_spin_unlock(&dest->jmp_lock);
+ /*
+ * The only possibility is that the jump was unlinked via
+ * tb_jump_unlink(dest). Seeing here another destination would be a bug,
+ * because we set the LSB above.
+ */
+ g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID);
+ return;
+ }
+ /*
+ * We first acquired the lock, and since the destination pointer matches,
+ * we know for sure that @orig is in the jmp list.
+ */
+ pprev = &dest->jmp_list_head;
+ TB_FOR_EACH_JMP(dest, tb, n) {
+ if (tb == orig && n == n_orig) {
+ *pprev = tb->jmp_list_next[n];
+ /* no need to set orig->jmp_dest[n]; setting the LSB was enough */
+ qemu_spin_unlock(&dest->jmp_lock);
+ return;
+ }
+ pprev = &tb->jmp_list_next[n];
+ }
+ g_assert_not_reached();
+}
+
+/*
+ * Reset the jump entry 'n' of a TB so that it is not chained to another TB.
+ */
+void tb_reset_jump(TranslationBlock *tb, int n)
+{
+ uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]);
+ tb_set_jmp_target(tb, n, addr);
+}
+
+/* remove any jumps to the TB */
+static inline void tb_jmp_unlink(TranslationBlock *dest)
+{
+ TranslationBlock *tb;
+ int n;
+
+ qemu_spin_lock(&dest->jmp_lock);
+
+ TB_FOR_EACH_JMP(dest, tb, n) {
+ tb_reset_jump(tb, n);
+ qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
+ /* No need to clear the list entry; setting the dest ptr is enough */
+ }
+ dest->jmp_list_head = (uintptr_t)NULL;
+
+ qemu_spin_unlock(&dest->jmp_lock);
+}
+
+static void tb_jmp_cache_inval_tb(TranslationBlock *tb)
+{
+ CPUState *cpu;
+
+ if (TARGET_TB_PCREL) {
+ /* A TB may be at any virtual address */
+ CPU_FOREACH(cpu) {
+ tcg_flush_jmp_cache(cpu);
+ }
+ } else {
+ uint32_t h = tb_jmp_cache_hash_func(tb_pc(tb));
+
+ CPU_FOREACH(cpu) {
+ CPUJumpCache *jc = cpu->tb_jmp_cache;
+
+ if (qatomic_read(&jc->array[h].tb) == tb) {
+ qatomic_set(&jc->array[h].tb, NULL);
+ }
+ }
+ }
+}
+
+/*
+ * In user-mode, call with mmap_lock held.
+ * In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
+ * locks held.
+ */
+static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
+{
+ PageDesc *p;
+ uint32_t h;
+ tb_page_addr_t phys_pc;
+ uint32_t orig_cflags = tb_cflags(tb);
+
+ assert_memory_lock();
+
+ /* make sure no further incoming jumps will be chained to this TB */
+ qemu_spin_lock(&tb->jmp_lock);
+ qatomic_set(&tb->cflags, tb->cflags | CF_INVALID);
+ qemu_spin_unlock(&tb->jmp_lock);
+
+ /* remove the TB from the hash list */
+ phys_pc = tb_page_addr0(tb);
+ h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)),
+ tb->flags, orig_cflags, tb->trace_vcpu_dstate);
+ if (!qht_remove(&tb_ctx.htable, tb, h)) {
+ return;
+ }
+
+ /* remove the TB from the page list */
+ if (rm_from_page_list) {
+ p = page_find(phys_pc >> TARGET_PAGE_BITS);
+ tb_page_remove(p, tb);
+ phys_pc = tb_page_addr1(tb);
+ if (phys_pc != -1) {
+ p = page_find(phys_pc >> TARGET_PAGE_BITS);
+ tb_page_remove(p, tb);
+ }
+ }
+
+ /* remove the TB from the hash list */
+ tb_jmp_cache_inval_tb(tb);
+
+ /* suppress this TB from the two jump lists */
+ tb_remove_from_jmp_list(tb, 0);
+ tb_remove_from_jmp_list(tb, 1);
+
+ /* suppress any remaining jumps to this TB */
+ tb_jmp_unlink(tb);
+
+ qatomic_set(&tb_ctx.tb_phys_invalidate_count,
+ tb_ctx.tb_phys_invalidate_count + 1);
+}
+
+static void tb_phys_invalidate__locked(TranslationBlock *tb)
+{
+ qemu_thread_jit_write();
+ do_tb_phys_invalidate(tb, true);
+ qemu_thread_jit_execute();
+}
+
+static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
+ PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc)
+{
+ PageDesc *p1, *p2;
+ tb_page_addr_t page1;
+ tb_page_addr_t page2;
+
+ assert_memory_lock();
+ g_assert(phys1 != -1);
+
+ page1 = phys1 >> TARGET_PAGE_BITS;
+ page2 = phys2 >> TARGET_PAGE_BITS;
+
+ p1 = page_find_alloc(page1, alloc);
+ if (ret_p1) {
+ *ret_p1 = p1;
+ }
+ if (likely(phys2 == -1)) {
+ page_lock(p1);
+ return;
+ } else if (page1 == page2) {
+ page_lock(p1);
+ if (ret_p2) {
+ *ret_p2 = p1;
+ }
+ return;
+ }
+ p2 = page_find_alloc(page2, alloc);
+ if (ret_p2) {
+ *ret_p2 = p2;
+ }
+ if (page1 < page2) {
+ page_lock(p1);
+ page_lock(p2);
+ } else {
+ page_lock(p2);
+ page_lock(p1);
+ }
+}
+
+#ifdef CONFIG_USER_ONLY
+static inline void page_lock_tb(const TranslationBlock *tb) { }
+static inline void page_unlock_tb(const TranslationBlock *tb) { }
+#else
+/* lock the page(s) of a TB in the correct acquisition order */
+static void page_lock_tb(const TranslationBlock *tb)
+{
+ page_lock_pair(NULL, tb_page_addr0(tb), NULL, tb_page_addr1(tb), false);
+}
+
+static void page_unlock_tb(const TranslationBlock *tb)
+{
+ PageDesc *p1 = page_find(tb_page_addr0(tb) >> TARGET_PAGE_BITS);
+
+ page_unlock(p1);
+ if (unlikely(tb_page_addr1(tb) != -1)) {
+ PageDesc *p2 = page_find(tb_page_addr1(tb) >> TARGET_PAGE_BITS);
+
+ if (p2 != p1) {
+ page_unlock(p2);
+ }
+ }
+}
+#endif
+
+/*
+ * Invalidate one TB.
+ * Called with mmap_lock held in user-mode.
+ */
+void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
+{
+ if (page_addr == -1 && tb_page_addr0(tb) != -1) {
+ page_lock_tb(tb);
+ do_tb_phys_invalidate(tb, true);
+ page_unlock_tb(tb);
+ } else {
+ do_tb_phys_invalidate(tb, false);
+ }
+}
+
+/*
+ * Add the tb in the target page and protect it if necessary.
+ * Called with mmap_lock held for user-mode emulation.
+ * Called with @p->lock held in !user-mode.
+ */
+static inline void tb_page_add(PageDesc *p, TranslationBlock *tb,
+ unsigned int n, tb_page_addr_t page_addr)
+{
+#ifndef CONFIG_USER_ONLY
+ bool page_already_protected;
+#endif
+
+ assert_page_locked(p);
+
+ tb->page_next[n] = p->first_tb;
+#ifndef CONFIG_USER_ONLY
+ page_already_protected = p->first_tb != (uintptr_t)NULL;
+#endif
+ p->first_tb = (uintptr_t)tb | n;
+
+#if defined(CONFIG_USER_ONLY)
+ /* translator_loop() must have made all TB pages non-writable */
+ assert(!(p->flags & PAGE_WRITE));
+#else
+ /*
+ * If some code is already present, then the pages are already
+ * protected. So we handle the case where only the first TB is
+ * allocated in a physical page.
+ */
+ if (!page_already_protected) {
+ tlb_protect_code(page_addr);
+ }
+#endif
+}
+
+/*
+ * Add a new TB and link it to the physical page tables. phys_page2 is
+ * (-1) to indicate that only one page contains the TB.
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ *
+ * Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
+ * Note that in !user-mode, another thread might have already added a TB
+ * for the same block of guest code that @tb corresponds to. In that case,
+ * the caller should discard the original @tb, and use instead the returned TB.
+ */
+TranslationBlock *tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+ tb_page_addr_t phys_page2)
+{
+ PageDesc *p;
+ PageDesc *p2 = NULL;
+ void *existing_tb = NULL;
+ uint32_t h;
+
+ assert_memory_lock();
+ tcg_debug_assert(!(tb->cflags & CF_INVALID));
+
+ /*
+ * Add the TB to the page list, acquiring first the pages's locks.
+ * We keep the locks held until after inserting the TB in the hash table,
+ * so that if the insertion fails we know for sure that the TBs are still
+ * in the page descriptors.
+ * Note that inserting into the hash table first isn't an option, since
+ * we can only insert TBs that are fully initialized.
+ */
+ page_lock_pair(&p, phys_pc, &p2, phys_page2, true);
+ tb_page_add(p, tb, 0, phys_pc);
+ if (p2) {
+ tb_page_add(p2, tb, 1, phys_page2);
+ }
+
+ /* add in the hash table */
+ h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)),
+ tb->flags, tb->cflags, tb->trace_vcpu_dstate);
+ qht_insert(&tb_ctx.htable, tb, h, &existing_tb);
+
+ /* remove TB from the page(s) if we couldn't insert it */
+ if (unlikely(existing_tb)) {
+ tb_page_remove(p, tb);
+ if (p2) {
+ tb_page_remove(p2, tb);
+ }
+ tb = existing_tb;
+ }
+
+ if (p2 && p2 != p) {
+ page_unlock(p2);
+ }
+ page_unlock(p);
+ return tb;
+}
+
+/*
+ * @p must be non-NULL.
+ * user-mode: call with mmap_lock held.
+ * !user-mode: call with all @pages locked.
+ */
+static void
+tb_invalidate_phys_page_range__locked(struct page_collection *pages,
+ PageDesc *p, tb_page_addr_t start,
+ tb_page_addr_t end,
+ uintptr_t retaddr)
+{
+ TranslationBlock *tb;
+ tb_page_addr_t tb_start, tb_end;
+ int n;
+#ifdef TARGET_HAS_PRECISE_SMC
+ CPUState *cpu = current_cpu;
+ bool current_tb_not_found = retaddr != 0;
+ bool current_tb_modified = false;
+ TranslationBlock *current_tb = NULL;
+#endif /* TARGET_HAS_PRECISE_SMC */
+
+ assert_page_locked(p);
+
+ /*
+ * We remove all the TBs in the range [start, end[.
+ * XXX: see if in some cases it could be faster to invalidate all the code
+ */
+ PAGE_FOR_EACH_TB(p, tb, n) {
+ assert_page_locked(p);
+ /* NOTE: this is subtle as a TB may span two physical pages */
+ if (n == 0) {
+ /* NOTE: tb_end may be after the end of the page, but
+ it is not a problem */
+ tb_start = tb_page_addr0(tb);
+ tb_end = tb_start + tb->size;
+ } else {
+ tb_start = tb_page_addr1(tb);
+ tb_end = tb_start + ((tb_page_addr0(tb) + tb->size)
+ & ~TARGET_PAGE_MASK);
+ }
+ if (!(tb_end <= start || tb_start >= end)) {
+#ifdef TARGET_HAS_PRECISE_SMC
+ if (current_tb_not_found) {
+ current_tb_not_found = false;
+ /* now we have a real cpu fault */
+ current_tb = tcg_tb_lookup(retaddr);
+ }
+ if (current_tb == tb &&
+ (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
+ /*
+ * If we are modifying the current TB, we must stop
+ * its execution. We could be more precise by checking
+ * that the modification is after the current PC, but it
+ * would require a specialized function to partially
+ * restore the CPU state.
+ */
+ current_tb_modified = true;
+ cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
+ }
+#endif /* TARGET_HAS_PRECISE_SMC */
+ tb_phys_invalidate__locked(tb);
+ }
+ }
+#if !defined(CONFIG_USER_ONLY)
+ /* if no code remaining, no need to continue to use slow writes */
+ if (!p->first_tb) {
+ tlb_unprotect_code(start);
+ }
+#endif
+#ifdef TARGET_HAS_PRECISE_SMC
+ if (current_tb_modified) {
+ page_collection_unlock(pages);
+ /* Force execution of one insn next time. */
+ cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
+ mmap_unlock();
+ cpu_loop_exit_noexc(cpu);
+ }
+#endif
+}
+
+/*
+ * Invalidate all TBs which intersect with the target physical
+ * address page @addr.
+ *
+ * Called with mmap_lock held for user-mode emulation
+ */
+void tb_invalidate_phys_page(tb_page_addr_t addr)
+{
+ struct page_collection *pages;
+ tb_page_addr_t start, end;
+ PageDesc *p;
+
+ assert_memory_lock();
+
+ p = page_find(addr >> TARGET_PAGE_BITS);
+ if (p == NULL) {
+ return;
+ }
+
+ start = addr & TARGET_PAGE_MASK;
+ end = start + TARGET_PAGE_SIZE;
+ pages = page_collection_lock(start, end);
+ tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
+ page_collection_unlock(pages);
+}
+
+/*
+ * Invalidate all TBs which intersect with the target physical address range
+ * [start;end[. NOTE: start and end may refer to *different* physical pages.
+ * 'is_cpu_write_access' should be true if called from a real cpu write
+ * access: the virtual CPU will exit the current TB if code is modified inside
+ * this TB.
+ *
+ * Called with mmap_lock held for user-mode emulation.
+ */
+void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end)
+{
+ struct page_collection *pages;
+ tb_page_addr_t next;
+
+ assert_memory_lock();
+
+ pages = page_collection_lock(start, end);
+ for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
+ start < end;
+ start = next, next += TARGET_PAGE_SIZE) {
+ PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
+ tb_page_addr_t bound = MIN(next, end);
+
+ if (pd == NULL) {
+ continue;
+ }
+ tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
+ }
+ page_collection_unlock(pages);
+}
+
+#ifdef CONFIG_SOFTMMU
+/*
+ * len must be <= 8 and start must be a multiple of len.
+ * Called via softmmu_template.h when code areas are written to with
+ * iothread mutex not held.
+ *
+ * Call with all @pages in the range [@start, @start + len[ locked.
+ */
+void tb_invalidate_phys_page_fast(struct page_collection *pages,
+ tb_page_addr_t start, int len,
+ uintptr_t retaddr)
+{
+ PageDesc *p;
+
+ assert_memory_lock();
+
+ p = page_find(start >> TARGET_PAGE_BITS);
+ if (!p) {
+ return;
+ }
+
+ assert_page_locked(p);
+ tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
+ retaddr);
+}
+#else
+/*
+ * Called with mmap_lock held. If pc is not 0 then it indicates the
+ * host PC of the faulting store instruction that caused this invalidate.
+ * Returns true if the caller needs to abort execution of the current
+ * TB (because it was modified by this store and the guest CPU has
+ * precise-SMC semantics).
+ */
+bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc)
+{
+ TranslationBlock *tb;
+ PageDesc *p;
+ int n;
+#ifdef TARGET_HAS_PRECISE_SMC
+ TranslationBlock *current_tb = NULL;
+ CPUState *cpu = current_cpu;
+ bool current_tb_modified = false;
+#endif
+
+ assert_memory_lock();
+
+ addr &= TARGET_PAGE_MASK;
+ p = page_find(addr >> TARGET_PAGE_BITS);
+ if (!p) {
+ return false;
+ }
+
+#ifdef TARGET_HAS_PRECISE_SMC
+ if (p->first_tb && pc != 0) {
+ current_tb = tcg_tb_lookup(pc);
+ }
+#endif
+ assert_page_locked(p);
+ PAGE_FOR_EACH_TB(p, tb, n) {
+#ifdef TARGET_HAS_PRECISE_SMC
+ if (current_tb == tb &&
+ (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
+ /*
+ * If we are modifying the current TB, we must stop its execution.
+ * We could be more precise by checking that the modification is
+ * after the current PC, but it would require a specialized
+ * function to partially restore the CPU state.
+ */
+ current_tb_modified = true;
+ cpu_restore_state_from_tb(cpu, current_tb, pc, true);
+ }
+#endif /* TARGET_HAS_PRECISE_SMC */
+ tb_phys_invalidate(tb, addr);
+ }
+ p->first_tb = (uintptr_t)NULL;
+#ifdef TARGET_HAS_PRECISE_SMC
+ if (current_tb_modified) {
+ /* Force execution of one insn next time. */
+ cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
+ return true;
+ }
+#endif
+
+ return false;
+}
+#endif
diff --git a/accel/tcg/tcg-accel-ops-mttcg.c b/accel/tcg/tcg-accel-ops-mttcg.c
index ba997f6cfe..d50239e0e2 100644
--- a/accel/tcg/tcg-accel-ops-mttcg.c
+++ b/accel/tcg/tcg-accel-ops-mttcg.c
@@ -70,8 +70,6 @@ static void *mttcg_cpu_thread_fn(void *arg)
assert(tcg_enabled());
g_assert(!icount_enabled());
- tcg_cpu_init_cflags(cpu, current_machine->smp.max_cpus > 1);
-
rcu_register_thread();
force_rcu.notifier.notify = mttcg_force_rcu;
force_rcu.cpu = cpu;
@@ -141,6 +139,9 @@ void mttcg_start_vcpu_thread(CPUState *cpu)
{
char thread_name[VCPU_THREAD_NAME_SIZE];
+ g_assert(tcg_enabled());
+ tcg_cpu_init_cflags(cpu, current_machine->smp.max_cpus > 1);
+
cpu->thread = g_new0(QemuThread, 1);
cpu->halt_cond = g_malloc0(sizeof(QemuCond));
qemu_cond_init(cpu->halt_cond);
diff --git a/accel/tcg/tcg-accel-ops-rr.c b/accel/tcg/tcg-accel-ops-rr.c
index cc912df108..290833a37f 100644
--- a/accel/tcg/tcg-accel-ops-rr.c
+++ b/accel/tcg/tcg-accel-ops-rr.c
@@ -152,9 +152,7 @@ static void *rr_cpu_thread_fn(void *arg)
Notifier force_rcu;
CPUState *cpu = arg;
- g_assert(tcg_enabled());
- tcg_cpu_init_cflags(cpu, false);
-
+ assert(tcg_enabled());
rcu_register_thread();
force_rcu.notify = rr_force_rcu;
rcu_add_force_rcu_notifier(&force_rcu);
@@ -277,6 +275,9 @@ void rr_start_vcpu_thread(CPUState *cpu)
static QemuCond *single_tcg_halt_cond;
static QemuThread *single_tcg_cpu_thread;
+ g_assert(tcg_enabled());
+ tcg_cpu_init_cflags(cpu, false);
+
if (!single_tcg_cpu_thread) {
cpu->thread = g_new0(QemuThread, 1);
cpu->halt_cond = g_new0(QemuCond, 1);
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index 4ed75a13e1..f185356a36 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -63,57 +63,7 @@
#include "tb-context.h"
#include "internal.h"
-/* #define DEBUG_TB_INVALIDATE */
-/* #define DEBUG_TB_FLUSH */
/* make various TB consistency checks */
-/* #define DEBUG_TB_CHECK */
-
-#ifdef DEBUG_TB_INVALIDATE
-#define DEBUG_TB_INVALIDATE_GATE 1
-#else
-#define DEBUG_TB_INVALIDATE_GATE 0
-#endif
-
-#ifdef DEBUG_TB_FLUSH
-#define DEBUG_TB_FLUSH_GATE 1
-#else
-#define DEBUG_TB_FLUSH_GATE 0
-#endif
-
-#if !defined(CONFIG_USER_ONLY)
-/* TB consistency checks only implemented for usermode emulation. */
-#undef DEBUG_TB_CHECK
-#endif
-
-#ifdef DEBUG_TB_CHECK
-#define DEBUG_TB_CHECK_GATE 1
-#else
-#define DEBUG_TB_CHECK_GATE 0
-#endif
-
-/* Access to the various translations structures need to be serialised via locks
- * for consistency.
- * In user-mode emulation access to the memory related structures are protected
- * with mmap_lock.
- * In !user-mode we use per-page locks.
- */
-#ifdef CONFIG_SOFTMMU
-#define assert_memory_lock()
-#else
-#define assert_memory_lock() tcg_debug_assert(have_mmap_lock())
-#endif
-
-typedef struct PageDesc {
- /* list of TBs intersecting this ram page */
- uintptr_t first_tb;
-#ifdef CONFIG_USER_ONLY
- unsigned long flags;
- void *target_data;
-#endif
-#ifdef CONFIG_SOFTMMU
- QemuSpin lock;
-#endif
-} PageDesc;
/**
* struct page_entry - page descriptor entry
@@ -159,18 +109,6 @@ struct page_collection {
struct page_entry *max;
};
-/* list iterators for lists of tagged pointers in TranslationBlock */
-#define TB_FOR_EACH_TAGGED(head, tb, n, field) \
- for (n = (head) & 1, tb = (TranslationBlock *)((head) & ~1); \
- tb; tb = (TranslationBlock *)tb->field[n], n = (uintptr_t)tb & 1, \
- tb = (TranslationBlock *)((uintptr_t)tb & ~1))
-
-#define PAGE_FOR_EACH_TB(pagedesc, tb, n) \
- TB_FOR_EACH_TAGGED((pagedesc)->first_tb, tb, n, page_next)
-
-#define TB_FOR_EACH_JMP(head_tb, tb, n) \
- TB_FOR_EACH_TAGGED((head_tb)->jmp_list_head, tb, n, jmp_list_next)
-
/*
* In system mode we want L1_MAP to be based on ram offsets,
* while in user mode we want it to be based on virtual addresses.
@@ -188,10 +126,6 @@ struct page_collection {
# define L1_MAP_ADDR_SPACE_BITS MIN(HOST_LONG_BITS, TARGET_ABI_BITS)
#endif
-/* Size of the L2 (and L3, etc) page tables. */
-#define V_L2_BITS 10
-#define V_L2_SIZE (1 << V_L2_BITS)
-
/* Make sure all possible CPU event bits fit in tb->trace_vcpu_dstate */
QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
sizeof_field(TranslationBlock, trace_vcpu_dstate)
@@ -200,18 +134,11 @@ QEMU_BUILD_BUG_ON(CPU_TRACE_DSTATE_MAX_EVENTS >
/*
* L1 Mapping properties
*/
-static int v_l1_size;
-static int v_l1_shift;
-static int v_l2_levels;
+int v_l1_size;
+int v_l1_shift;
+int v_l2_levels;
-/* The bottom level has pointers to PageDesc, and is indexed by
- * anything from 4 to (V_L2_BITS + 3) bits, depending on target page size.
- */
-#define V_L1_MIN_BITS 4
-#define V_L1_MAX_BITS (V_L2_BITS + 3)
-#define V_L1_MAX_SIZE (1 << V_L1_MAX_BITS)
-
-static void *l1_map[V_L1_MAX_SIZE];
+void *l1_map[V_L1_MAX_SIZE];
TBContext tb_ctx;
@@ -324,12 +251,11 @@ static int encode_search(TranslationBlock *tb, uint8_t *block)
* When reset_icount is true, current TB will be interrupted and
* icount should be recalculated.
*/
-static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
- uintptr_t searched_pc, bool reset_icount)
+int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
+ uintptr_t searched_pc, bool reset_icount)
{
- target_ulong data[TARGET_INSN_START_WORDS];
+ uint64_t data[TARGET_INSN_START_WORDS];
uintptr_t host_pc = (uintptr_t)tb->tc.ptr;
- CPUArchState *env = cpu->env_ptr;
const uint8_t *p = tb->tc.ptr + tb->tc.size;
int i, j, num_insns = tb->icount;
#ifdef CONFIG_PROFILER
@@ -368,7 +294,8 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
and shift if to the number of actually executed instructions */
cpu_neg(cpu)->icount_decr.u16.low += num_insns - i;
}
- restore_state_to_opc(env, tb, data);
+
+ cpu->cc->tcg_ops->restore_state_to_opc(cpu, tb, data);
#ifdef CONFIG_PROFILER
qatomic_set(&prof->restore_time,
@@ -381,6 +308,14 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
bool cpu_restore_state(CPUState *cpu, uintptr_t host_pc, bool will_exit)
{
/*
+ * The pc update associated with restore without exit will
+ * break the relative pc adjustments performed by TARGET_TB_PCREL.
+ */
+ if (TARGET_TB_PCREL) {
+ assert(will_exit);
+ }
+
+ /*
* The host_pc has to be in the rx region of the code buffer.
* If it is not we will not be able to resolve it here.
* The two cases where host_pc will not be correct are:
@@ -471,7 +406,7 @@ void page_init(void)
#endif
}
-static PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc)
+PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc)
{
PageDesc *pd;
void **lp;
@@ -537,31 +472,8 @@ static PageDesc *page_find_alloc(tb_page_addr_t index, bool alloc)
return pd + (index & (V_L2_SIZE - 1));
}
-static inline PageDesc *page_find(tb_page_addr_t index)
-{
- return page_find_alloc(index, false);
-}
-
-static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
- PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc);
-
/* In user-mode page locks aren't used; mmap_lock is enough */
#ifdef CONFIG_USER_ONLY
-
-#define assert_page_locked(pd) tcg_debug_assert(have_mmap_lock())
-
-static inline void page_lock(PageDesc *pd)
-{ }
-
-static inline void page_unlock(PageDesc *pd)
-{ }
-
-static inline void page_lock_tb(const TranslationBlock *tb)
-{ }
-
-static inline void page_unlock_tb(const TranslationBlock *tb)
-{ }
-
struct page_collection *
page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
{
@@ -610,8 +522,7 @@ static void page_unlock__debug(const PageDesc *pd)
g_assert(removed);
}
-static void
-do_assert_page_locked(const PageDesc *pd, const char *file, int line)
+void do_assert_page_locked(const PageDesc *pd, const char *file, int line)
{
if (unlikely(!page_is_locked(pd))) {
error_report("assert_page_lock: PageDesc %p not locked @ %s:%d",
@@ -620,8 +531,6 @@ do_assert_page_locked(const PageDesc *pd, const char *file, int line)
}
}
-#define assert_page_locked(pd) do_assert_page_locked(pd, __FILE__, __LINE__)
-
void assert_no_pages_locked(void)
{
ht_pages_locked_debug_init();
@@ -630,50 +539,23 @@ void assert_no_pages_locked(void)
#else /* !CONFIG_DEBUG_TCG */
-#define assert_page_locked(pd)
-
-static inline void page_lock__debug(const PageDesc *pd)
-{
-}
-
-static inline void page_unlock__debug(const PageDesc *pd)
-{
-}
+static inline void page_lock__debug(const PageDesc *pd) { }
+static inline void page_unlock__debug(const PageDesc *pd) { }
#endif /* CONFIG_DEBUG_TCG */
-static inline void page_lock(PageDesc *pd)
+void page_lock(PageDesc *pd)
{
page_lock__debug(pd);
qemu_spin_lock(&pd->lock);
}
-static inline void page_unlock(PageDesc *pd)
+void page_unlock(PageDesc *pd)
{
qemu_spin_unlock(&pd->lock);
page_unlock__debug(pd);
}
-/* lock the page(s) of a TB in the correct acquisition order */
-static inline void page_lock_tb(const TranslationBlock *tb)
-{
- page_lock_pair(NULL, tb->page_addr[0], NULL, tb->page_addr[1], false);
-}
-
-static inline void page_unlock_tb(const TranslationBlock *tb)
-{
- PageDesc *p1 = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
-
- page_unlock(p1);
- if (unlikely(tb->page_addr[1] != -1)) {
- PageDesc *p2 = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
-
- if (p2 != p1) {
- page_unlock(p2);
- }
- }
-}
-
static inline struct page_entry *
page_entry_new(PageDesc *pd, tb_page_addr_t index)
{
@@ -824,9 +706,9 @@ page_collection_lock(tb_page_addr_t start, tb_page_addr_t end)
}
assert_page_locked(pd);
PAGE_FOR_EACH_TB(pd, tb, n) {
- if (page_trylock_add(set, tb->page_addr[0]) ||
- (tb->page_addr[1] != -1 &&
- page_trylock_add(set, tb->page_addr[1]))) {
+ if (page_trylock_add(set, tb_page_addr0(tb)) ||
+ (tb_page_addr1(tb) != -1 &&
+ page_trylock_add(set, tb_page_addr1(tb)))) {
/* drop all locks, and reacquire in order */
g_tree_foreach(set->tree, page_entry_unlock, NULL);
goto retry;
@@ -845,509 +727,6 @@ void page_collection_unlock(struct page_collection *set)
#endif /* !CONFIG_USER_ONLY */
-static void page_lock_pair(PageDesc **ret_p1, tb_page_addr_t phys1,
- PageDesc **ret_p2, tb_page_addr_t phys2, bool alloc)
-{
- PageDesc *p1, *p2;
- tb_page_addr_t page1;
- tb_page_addr_t page2;
-
- assert_memory_lock();
- g_assert(phys1 != -1);
-
- page1 = phys1 >> TARGET_PAGE_BITS;
- page2 = phys2 >> TARGET_PAGE_BITS;
-
- p1 = page_find_alloc(page1, alloc);
- if (ret_p1) {
- *ret_p1 = p1;
- }
- if (likely(phys2 == -1)) {
- page_lock(p1);
- return;
- } else if (page1 == page2) {
- page_lock(p1);
- if (ret_p2) {
- *ret_p2 = p1;
- }
- return;
- }
- p2 = page_find_alloc(page2, alloc);
- if (ret_p2) {
- *ret_p2 = p2;
- }
- if (page1 < page2) {
- page_lock(p1);
- page_lock(p2);
- } else {
- page_lock(p2);
- page_lock(p1);
- }
-}
-
-static bool tb_cmp(const void *ap, const void *bp)
-{
- const TranslationBlock *a = ap;
- const TranslationBlock *b = bp;
-
- return ((TARGET_TB_PCREL || tb_pc(a) == tb_pc(b)) &&
- a->cs_base == b->cs_base &&
- a->flags == b->flags &&
- (tb_cflags(a) & ~CF_INVALID) == (tb_cflags(b) & ~CF_INVALID) &&
- a->trace_vcpu_dstate == b->trace_vcpu_dstate &&
- a->page_addr[0] == b->page_addr[0] &&
- a->page_addr[1] == b->page_addr[1]);
-}
-
-void tb_htable_init(void)
-{
- unsigned int mode = QHT_MODE_AUTO_RESIZE;
-
- qht_init(&tb_ctx.htable, tb_cmp, CODE_GEN_HTABLE_SIZE, mode);
-}
-
-/* Set to NULL all the 'first_tb' fields in all PageDescs. */
-static void page_flush_tb_1(int level, void **lp)
-{
- int i;
-
- if (*lp == NULL) {
- return;
- }
- if (level == 0) {
- PageDesc *pd = *lp;
-
- for (i = 0; i < V_L2_SIZE; ++i) {
- page_lock(&pd[i]);
- pd[i].first_tb = (uintptr_t)NULL;
- page_unlock(&pd[i]);
- }
- } else {
- void **pp = *lp;
-
- for (i = 0; i < V_L2_SIZE; ++i) {
- page_flush_tb_1(level - 1, pp + i);
- }
- }
-}
-
-static void page_flush_tb(void)
-{
- int i, l1_sz = v_l1_size;
-
- for (i = 0; i < l1_sz; i++) {
- page_flush_tb_1(v_l2_levels, l1_map + i);
- }
-}
-
-static gboolean tb_host_size_iter(gpointer key, gpointer value, gpointer data)
-{
- const TranslationBlock *tb = value;
- size_t *size = data;
-
- *size += tb->tc.size;
- return false;
-}
-
-/* flush all the translation blocks */
-static void do_tb_flush(CPUState *cpu, run_on_cpu_data tb_flush_count)
-{
- bool did_flush = false;
-
- mmap_lock();
- /* If it is already been done on request of another CPU,
- * just retry.
- */
- if (tb_ctx.tb_flush_count != tb_flush_count.host_int) {
- goto done;
- }
- did_flush = true;
-
- if (DEBUG_TB_FLUSH_GATE) {
- size_t nb_tbs = tcg_nb_tbs();
- size_t host_size = 0;
-
- tcg_tb_foreach(tb_host_size_iter, &host_size);
- printf("qemu: flush code_size=%zu nb_tbs=%zu avg_tb_size=%zu\n",
- tcg_code_size(), nb_tbs, nb_tbs > 0 ? host_size / nb_tbs : 0);
- }
-
- CPU_FOREACH(cpu) {
- tcg_flush_jmp_cache(cpu);
- }
-
- qht_reset_size(&tb_ctx.htable, CODE_GEN_HTABLE_SIZE);
- page_flush_tb();
-
- tcg_region_reset_all();
- /* XXX: flush processor icache at this point if cache flush is
- expensive */
- qatomic_mb_set(&tb_ctx.tb_flush_count, tb_ctx.tb_flush_count + 1);
-
-done:
- mmap_unlock();
- if (did_flush) {
- qemu_plugin_flush_cb();
- }
-}
-
-void tb_flush(CPUState *cpu)
-{
- if (tcg_enabled()) {
- unsigned tb_flush_count = qatomic_mb_read(&tb_ctx.tb_flush_count);
-
- if (cpu_in_exclusive_context(cpu)) {
- do_tb_flush(cpu, RUN_ON_CPU_HOST_INT(tb_flush_count));
- } else {
- async_safe_run_on_cpu(cpu, do_tb_flush,
- RUN_ON_CPU_HOST_INT(tb_flush_count));
- }
- }
-}
-
-/*
- * Formerly ifdef DEBUG_TB_CHECK. These debug functions are user-mode-only,
- * so in order to prevent bit rot we compile them unconditionally in user-mode,
- * and let the optimizer get rid of them by wrapping their user-only callers
- * with if (DEBUG_TB_CHECK_GATE).
- */
-#ifdef CONFIG_USER_ONLY
-
-static void do_tb_invalidate_check(void *p, uint32_t hash, void *userp)
-{
- TranslationBlock *tb = p;
- target_ulong addr = *(target_ulong *)userp;
-
- if (!(addr + TARGET_PAGE_SIZE <= tb_pc(tb) ||
- addr >= tb_pc(tb) + tb->size)) {
- printf("ERROR invalidate: address=" TARGET_FMT_lx
- " PC=%08lx size=%04x\n", addr, (long)tb_pc(tb), tb->size);
- }
-}
-
-/* verify that all the pages have correct rights for code
- *
- * Called with mmap_lock held.
- */
-static void tb_invalidate_check(target_ulong address)
-{
- address &= TARGET_PAGE_MASK;
- qht_iter(&tb_ctx.htable, do_tb_invalidate_check, &address);
-}
-
-static void do_tb_page_check(void *p, uint32_t hash, void *userp)
-{
- TranslationBlock *tb = p;
- int flags1, flags2;
-
- flags1 = page_get_flags(tb_pc(tb));
- flags2 = page_get_flags(tb_pc(tb) + tb->size - 1);
- if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
- printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
- (long)tb_pc(tb), tb->size, flags1, flags2);
- }
-}
-
-/* verify that all the pages have correct rights for code */
-static void tb_page_check(void)
-{
- qht_iter(&tb_ctx.htable, do_tb_page_check, NULL);
-}
-
-#endif /* CONFIG_USER_ONLY */
-
-/*
- * user-mode: call with mmap_lock held
- * !user-mode: call with @pd->lock held
- */
-static inline void tb_page_remove(PageDesc *pd, TranslationBlock *tb)
-{
- TranslationBlock *tb1;
- uintptr_t *pprev;
- unsigned int n1;
-
- assert_page_locked(pd);
- pprev = &pd->first_tb;
- PAGE_FOR_EACH_TB(pd, tb1, n1) {
- if (tb1 == tb) {
- *pprev = tb1->page_next[n1];
- return;
- }
- pprev = &tb1->page_next[n1];
- }
- g_assert_not_reached();
-}
-
-/* remove @orig from its @n_orig-th jump list */
-static inline void tb_remove_from_jmp_list(TranslationBlock *orig, int n_orig)
-{
- uintptr_t ptr, ptr_locked;
- TranslationBlock *dest;
- TranslationBlock *tb;
- uintptr_t *pprev;
- int n;
-
- /* mark the LSB of jmp_dest[] so that no further jumps can be inserted */
- ptr = qatomic_or_fetch(&orig->jmp_dest[n_orig], 1);
- dest = (TranslationBlock *)(ptr & ~1);
- if (dest == NULL) {
- return;
- }
-
- qemu_spin_lock(&dest->jmp_lock);
- /*
- * While acquiring the lock, the jump might have been removed if the
- * destination TB was invalidated; check again.
- */
- ptr_locked = qatomic_read(&orig->jmp_dest[n_orig]);
- if (ptr_locked != ptr) {
- qemu_spin_unlock(&dest->jmp_lock);
- /*
- * The only possibility is that the jump was unlinked via
- * tb_jump_unlink(dest). Seeing here another destination would be a bug,
- * because we set the LSB above.
- */
- g_assert(ptr_locked == 1 && dest->cflags & CF_INVALID);
- return;
- }
- /*
- * We first acquired the lock, and since the destination pointer matches,
- * we know for sure that @orig is in the jmp list.
- */
- pprev = &dest->jmp_list_head;
- TB_FOR_EACH_JMP(dest, tb, n) {
- if (tb == orig && n == n_orig) {
- *pprev = tb->jmp_list_next[n];
- /* no need to set orig->jmp_dest[n]; setting the LSB was enough */
- qemu_spin_unlock(&dest->jmp_lock);
- return;
- }
- pprev = &tb->jmp_list_next[n];
- }
- g_assert_not_reached();
-}
-
-/* reset the jump entry 'n' of a TB so that it is not chained to
- another TB */
-static inline void tb_reset_jump(TranslationBlock *tb, int n)
-{
- uintptr_t addr = (uintptr_t)(tb->tc.ptr + tb->jmp_reset_offset[n]);
- tb_set_jmp_target(tb, n, addr);
-}
-
-/* remove any jumps to the TB */
-static inline void tb_jmp_unlink(TranslationBlock *dest)
-{
- TranslationBlock *tb;
- int n;
-
- qemu_spin_lock(&dest->jmp_lock);
-
- TB_FOR_EACH_JMP(dest, tb, n) {
- tb_reset_jump(tb, n);
- qatomic_and(&tb->jmp_dest[n], (uintptr_t)NULL | 1);
- /* No need to clear the list entry; setting the dest ptr is enough */
- }
- dest->jmp_list_head = (uintptr_t)NULL;
-
- qemu_spin_unlock(&dest->jmp_lock);
-}
-
-static void tb_jmp_cache_inval_tb(TranslationBlock *tb)
-{
- CPUState *cpu;
-
- if (TARGET_TB_PCREL) {
- /* A TB may be at any virtual address */
- CPU_FOREACH(cpu) {
- tcg_flush_jmp_cache(cpu);
- }
- } else {
- uint32_t h = tb_jmp_cache_hash_func(tb_pc(tb));
-
- CPU_FOREACH(cpu) {
- CPUJumpCache *jc = cpu->tb_jmp_cache;
-
- if (qatomic_read(&jc->array[h].tb) == tb) {
- qatomic_set(&jc->array[h].tb, NULL);
- }
- }
- }
-}
-
-/*
- * In user-mode, call with mmap_lock held.
- * In !user-mode, if @rm_from_page_list is set, call with the TB's pages'
- * locks held.
- */
-static void do_tb_phys_invalidate(TranslationBlock *tb, bool rm_from_page_list)
-{
- PageDesc *p;
- uint32_t h;
- tb_page_addr_t phys_pc;
- uint32_t orig_cflags = tb_cflags(tb);
-
- assert_memory_lock();
-
- /* make sure no further incoming jumps will be chained to this TB */
- qemu_spin_lock(&tb->jmp_lock);
- qatomic_set(&tb->cflags, tb->cflags | CF_INVALID);
- qemu_spin_unlock(&tb->jmp_lock);
-
- /* remove the TB from the hash list */
- phys_pc = tb->page_addr[0];
- h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)),
- tb->flags, orig_cflags, tb->trace_vcpu_dstate);
- if (!qht_remove(&tb_ctx.htable, tb, h)) {
- return;
- }
-
- /* remove the TB from the page list */
- if (rm_from_page_list) {
- p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
- tb_page_remove(p, tb);
- if (tb->page_addr[1] != -1) {
- p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
- tb_page_remove(p, tb);
- }
- }
-
- /* remove the TB from the hash list */
- tb_jmp_cache_inval_tb(tb);
-
- /* suppress this TB from the two jump lists */
- tb_remove_from_jmp_list(tb, 0);
- tb_remove_from_jmp_list(tb, 1);
-
- /* suppress any remaining jumps to this TB */
- tb_jmp_unlink(tb);
-
- qatomic_set(&tb_ctx.tb_phys_invalidate_count,
- tb_ctx.tb_phys_invalidate_count + 1);
-}
-
-static void tb_phys_invalidate__locked(TranslationBlock *tb)
-{
- qemu_thread_jit_write();
- do_tb_phys_invalidate(tb, true);
- qemu_thread_jit_execute();
-}
-
-/* invalidate one TB
- *
- * Called with mmap_lock held in user-mode.
- */
-void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr)
-{
- if (page_addr == -1 && tb->page_addr[0] != -1) {
- page_lock_tb(tb);
- do_tb_phys_invalidate(tb, true);
- page_unlock_tb(tb);
- } else {
- do_tb_phys_invalidate(tb, false);
- }
-}
-
-/* add the tb in the target page and protect it if necessary
- *
- * Called with mmap_lock held for user-mode emulation.
- * Called with @p->lock held in !user-mode.
- */
-static inline void tb_page_add(PageDesc *p, TranslationBlock *tb,
- unsigned int n, tb_page_addr_t page_addr)
-{
-#ifndef CONFIG_USER_ONLY
- bool page_already_protected;
-#endif
-
- assert_page_locked(p);
-
- tb->page_addr[n] = page_addr;
- tb->page_next[n] = p->first_tb;
-#ifndef CONFIG_USER_ONLY
- page_already_protected = p->first_tb != (uintptr_t)NULL;
-#endif
- p->first_tb = (uintptr_t)tb | n;
-
-#if defined(CONFIG_USER_ONLY)
- /* translator_loop() must have made all TB pages non-writable */
- assert(!(p->flags & PAGE_WRITE));
-#else
- /* if some code is already present, then the pages are already
- protected. So we handle the case where only the first TB is
- allocated in a physical page */
- if (!page_already_protected) {
- tlb_protect_code(page_addr);
- }
-#endif
-}
-
-/*
- * Add a new TB and link it to the physical page tables. phys_page2 is
- * (-1) to indicate that only one page contains the TB.
- *
- * Called with mmap_lock held for user-mode emulation.
- *
- * Returns a pointer @tb, or a pointer to an existing TB that matches @tb.
- * Note that in !user-mode, another thread might have already added a TB
- * for the same block of guest code that @tb corresponds to. In that case,
- * the caller should discard the original @tb, and use instead the returned TB.
- */
-static TranslationBlock *
-tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
- tb_page_addr_t phys_page2)
-{
- PageDesc *p;
- PageDesc *p2 = NULL;
- void *existing_tb = NULL;
- uint32_t h;
-
- assert_memory_lock();
- tcg_debug_assert(!(tb->cflags & CF_INVALID));
-
- /*
- * Add the TB to the page list, acquiring first the pages's locks.
- * We keep the locks held until after inserting the TB in the hash table,
- * so that if the insertion fails we know for sure that the TBs are still
- * in the page descriptors.
- * Note that inserting into the hash table first isn't an option, since
- * we can only insert TBs that are fully initialized.
- */
- page_lock_pair(&p, phys_pc, &p2, phys_page2, true);
- tb_page_add(p, tb, 0, phys_pc);
- if (p2) {
- tb_page_add(p2, tb, 1, phys_page2);
- } else {
- tb->page_addr[1] = -1;
- }
-
- /* add in the hash table */
- h = tb_hash_func(phys_pc, (TARGET_TB_PCREL ? 0 : tb_pc(tb)),
- tb->flags, tb->cflags, tb->trace_vcpu_dstate);
- qht_insert(&tb_ctx.htable, tb, h, &existing_tb);
-
- /* remove TB from the page(s) if we couldn't insert it */
- if (unlikely(existing_tb)) {
- tb_page_remove(p, tb);
- if (p2) {
- tb_page_remove(p2, tb);
- }
- tb = existing_tb;
- }
-
- if (p2 && p2 != p) {
- page_unlock(p2);
- }
- page_unlock(p);
-
-#ifdef CONFIG_USER_ONLY
- if (DEBUG_TB_CHECK_GATE) {
- tb_page_check();
- }
-#endif
- return tb;
-}
-
/* Called with mmap_lock held for user mode emulation. */
TranslationBlock *tb_gen_code(CPUState *cpu,
target_ulong pc, target_ulong cs_base,
@@ -1400,8 +779,8 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
tb->flags = flags;
tb->cflags = cflags;
tb->trace_vcpu_dstate = *cpu->trace_dstate;
- tb->page_addr[0] = phys_pc;
- tb->page_addr[1] = -1;
+ tb_set_page_addr0(tb, phys_pc);
+ tb_set_page_addr1(tb, -1);
tcg_ctx->tb_cflags = cflags;
tb_overflow:
@@ -1599,7 +978,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
* a temporary one-insn TB, and we have nothing left to do. Return early
* before attempting to link to other TBs or add to the lookup table.
*/
- if (tb->page_addr[0] == -1) {
+ if (tb_page_addr0(tb) == -1) {
return tb;
}
@@ -1614,7 +993,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
* No explicit memory barrier is required -- tb_link_page() makes the
* TB visible in a consistent state.
*/
- existing_tb = tb_link_page(tb, tb->page_addr[0], tb->page_addr[1]);
+ existing_tb = tb_link_page(tb, tb_page_addr0(tb), tb_page_addr1(tb));
/* if the TB already exists, discard what we just translated */
if (unlikely(existing_tb != tb)) {
uintptr_t orig_aligned = (uintptr_t)gen_code_buf;
@@ -1627,251 +1006,6 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
return tb;
}
-/*
- * @p must be non-NULL.
- * user-mode: call with mmap_lock held.
- * !user-mode: call with all @pages locked.
- */
-static void
-tb_invalidate_phys_page_range__locked(struct page_collection *pages,
- PageDesc *p, tb_page_addr_t start,
- tb_page_addr_t end,
- uintptr_t retaddr)
-{
- TranslationBlock *tb;
- tb_page_addr_t tb_start, tb_end;
- int n;
-#ifdef TARGET_HAS_PRECISE_SMC
- CPUState *cpu = current_cpu;
- CPUArchState *env = NULL;
- bool current_tb_not_found = retaddr != 0;
- bool current_tb_modified = false;
- TranslationBlock *current_tb = NULL;
- target_ulong current_pc = 0;
- target_ulong current_cs_base = 0;
- uint32_t current_flags = 0;
-#endif /* TARGET_HAS_PRECISE_SMC */
-
- assert_page_locked(p);
-
-#if defined(TARGET_HAS_PRECISE_SMC)
- if (cpu != NULL) {
- env = cpu->env_ptr;
- }
-#endif
-
- /* we remove all the TBs in the range [start, end[ */
- /* XXX: see if in some cases it could be faster to invalidate all
- the code */
- PAGE_FOR_EACH_TB(p, tb, n) {
- assert_page_locked(p);
- /* NOTE: this is subtle as a TB may span two physical pages */
- if (n == 0) {
- /* NOTE: tb_end may be after the end of the page, but
- it is not a problem */
- tb_start = tb->page_addr[0];
- tb_end = tb_start + tb->size;
- } else {
- tb_start = tb->page_addr[1];
- tb_end = tb_start + ((tb->page_addr[0] + tb->size)
- & ~TARGET_PAGE_MASK);
- }
- if (!(tb_end <= start || tb_start >= end)) {
-#ifdef TARGET_HAS_PRECISE_SMC
- if (current_tb_not_found) {
- current_tb_not_found = false;
- /* now we have a real cpu fault */
- current_tb = tcg_tb_lookup(retaddr);
- }
- if (current_tb == tb &&
- (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
- /*
- * If we are modifying the current TB, we must stop
- * its execution. We could be more precise by checking
- * that the modification is after the current PC, but it
- * would require a specialized function to partially
- * restore the CPU state.
- */
- current_tb_modified = true;
- cpu_restore_state_from_tb(cpu, current_tb, retaddr, true);
- cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
- &current_flags);
- }
-#endif /* TARGET_HAS_PRECISE_SMC */
- tb_phys_invalidate__locked(tb);
- }
- }
-#if !defined(CONFIG_USER_ONLY)
- /* if no code remaining, no need to continue to use slow writes */
- if (!p->first_tb) {
- tlb_unprotect_code(start);
- }
-#endif
-#ifdef TARGET_HAS_PRECISE_SMC
- if (current_tb_modified) {
- page_collection_unlock(pages);
- /* Force execution of one insn next time. */
- cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
- mmap_unlock();
- cpu_loop_exit_noexc(cpu);
- }
-#endif
-}
-
-/*
- * Invalidate all TBs which intersect with the target physical address range
- * [start;end[. NOTE: start and end must refer to the *same* physical page.
- * 'is_cpu_write_access' should be true if called from a real cpu write
- * access: the virtual CPU will exit the current TB if code is modified inside
- * this TB.
- *
- * Called with mmap_lock held for user-mode emulation
- */
-void tb_invalidate_phys_page_range(tb_page_addr_t start, tb_page_addr_t end)
-{
- struct page_collection *pages;
- PageDesc *p;
-
- assert_memory_lock();
-
- p = page_find(start >> TARGET_PAGE_BITS);
- if (p == NULL) {
- return;
- }
- pages = page_collection_lock(start, end);
- tb_invalidate_phys_page_range__locked(pages, p, start, end, 0);
- page_collection_unlock(pages);
-}
-
-/*
- * Invalidate all TBs which intersect with the target physical address range
- * [start;end[. NOTE: start and end may refer to *different* physical pages.
- * 'is_cpu_write_access' should be true if called from a real cpu write
- * access: the virtual CPU will exit the current TB if code is modified inside
- * this TB.
- *
- * Called with mmap_lock held for user-mode emulation.
- */
-#ifdef CONFIG_SOFTMMU
-void tb_invalidate_phys_range(ram_addr_t start, ram_addr_t end)
-#else
-void tb_invalidate_phys_range(target_ulong start, target_ulong end)
-#endif
-{
- struct page_collection *pages;
- tb_page_addr_t next;
-
- assert_memory_lock();
-
- pages = page_collection_lock(start, end);
- for (next = (start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
- start < end;
- start = next, next += TARGET_PAGE_SIZE) {
- PageDesc *pd = page_find(start >> TARGET_PAGE_BITS);
- tb_page_addr_t bound = MIN(next, end);
-
- if (pd == NULL) {
- continue;
- }
- tb_invalidate_phys_page_range__locked(pages, pd, start, bound, 0);
- }
- page_collection_unlock(pages);
-}
-
-#ifdef CONFIG_SOFTMMU
-/* len must be <= 8 and start must be a multiple of len.
- * Called via softmmu_template.h when code areas are written to with
- * iothread mutex not held.
- *
- * Call with all @pages in the range [@start, @start + len[ locked.
- */
-void tb_invalidate_phys_page_fast(struct page_collection *pages,
- tb_page_addr_t start, int len,
- uintptr_t retaddr)
-{
- PageDesc *p;
-
- assert_memory_lock();
-
- p = page_find(start >> TARGET_PAGE_BITS);
- if (!p) {
- return;
- }
-
- assert_page_locked(p);
- tb_invalidate_phys_page_range__locked(pages, p, start, start + len,
- retaddr);
-}
-#else
-/* Called with mmap_lock held. If pc is not 0 then it indicates the
- * host PC of the faulting store instruction that caused this invalidate.
- * Returns true if the caller needs to abort execution of the current
- * TB (because it was modified by this store and the guest CPU has
- * precise-SMC semantics).
- */
-static bool tb_invalidate_phys_page(tb_page_addr_t addr, uintptr_t pc)
-{
- TranslationBlock *tb;
- PageDesc *p;
- int n;
-#ifdef TARGET_HAS_PRECISE_SMC
- TranslationBlock *current_tb = NULL;
- CPUState *cpu = current_cpu;
- CPUArchState *env = NULL;
- int current_tb_modified = 0;
- target_ulong current_pc = 0;
- target_ulong current_cs_base = 0;
- uint32_t current_flags = 0;
-#endif
-
- assert_memory_lock();
-
- addr &= TARGET_PAGE_MASK;
- p = page_find(addr >> TARGET_PAGE_BITS);
- if (!p) {
- return false;
- }
-
-#ifdef TARGET_HAS_PRECISE_SMC
- if (p->first_tb && pc != 0) {
- current_tb = tcg_tb_lookup(pc);
- }
- if (cpu != NULL) {
- env = cpu->env_ptr;
- }
-#endif
- assert_page_locked(p);
- PAGE_FOR_EACH_TB(p, tb, n) {
-#ifdef TARGET_HAS_PRECISE_SMC
- if (current_tb == tb &&
- (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) {
- /* If we are modifying the current TB, we must stop
- its execution. We could be more precise by checking
- that the modification is after the current PC, but it
- would require a specialized function to partially
- restore the CPU state */
-
- current_tb_modified = 1;
- cpu_restore_state_from_tb(cpu, current_tb, pc, true);
- cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
- &current_flags);
- }
-#endif /* TARGET_HAS_PRECISE_SMC */
- tb_phys_invalidate(tb, addr);
- }
- p->first_tb = (uintptr_t)NULL;
-#ifdef TARGET_HAS_PRECISE_SMC
- if (current_tb_modified) {
- /* Force execution of one insn next time. */
- cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(cpu);
- return true;
- }
-#endif
-
- return false;
-}
-#endif
-
/* user-mode: call with mmap_lock held */
void tb_check_watchpoint(CPUState *cpu, uintptr_t retaddr)
{
@@ -2014,7 +1148,7 @@ static gboolean tb_tree_stats_iter(gpointer key, gpointer value, gpointer data)
if (tb->size > tst->max_target_size) {
tst->max_target_size = tb->size;
}
- if (tb->page_addr[1] != -1) {
+ if (tb_page_addr1(tb) != -1) {
tst->cross_page++;
}
if (tb->jmp_reset_offset[0] != TB_JMP_RESET_OFFSET_INVALID) {
@@ -2226,7 +1360,7 @@ int page_get_flags(target_ulong address)
void page_set_flags(target_ulong start, target_ulong end, int flags)
{
target_ulong addr, len;
- bool reset_target_data;
+ bool reset, inval_tb = false;
/* This function should never be called with addresses outside the
guest address space. If this assert fires, it probably indicates
@@ -2243,7 +1377,10 @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
if (flags & PAGE_WRITE) {
flags |= PAGE_WRITE_ORG;
}
- reset_target_data = !(flags & PAGE_VALID) || (flags & PAGE_RESET);
+ reset = !(flags & PAGE_VALID) || (flags & PAGE_RESET);
+ if (reset) {
+ page_reset_target_data(start, end);
+ }
flags &= ~PAGE_RESET;
for (addr = start, len = end - start;
@@ -2251,68 +1388,23 @@ void page_set_flags(target_ulong start, target_ulong end, int flags)
len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, true);
- /* If the write protection bit is set, then we invalidate
- the code inside. */
- if (!(p->flags & PAGE_WRITE) &&
- (flags & PAGE_WRITE) &&
- p->first_tb) {
- tb_invalidate_phys_page(addr, 0);
- }
- if (reset_target_data) {
- g_free(p->target_data);
- p->target_data = NULL;
- p->flags = flags;
- } else {
- /* Using mprotect on a page does not change sticky bits. */
- p->flags = (p->flags & PAGE_STICKY) | flags;
+ /*
+ * If the page was executable, but is reset, or is no longer
+ * executable, or has become writable, then invalidate any code.
+ */
+ if ((p->flags & PAGE_EXEC)
+ && (reset ||
+ !(flags & PAGE_EXEC) ||
+ (flags & ~p->flags & PAGE_WRITE))) {
+ inval_tb = true;
}
+ /* Using mprotect on a page does not change sticky bits. */
+ p->flags = (reset ? 0 : p->flags & PAGE_STICKY) | flags;
}
-}
-
-void page_reset_target_data(target_ulong start, target_ulong end)
-{
- target_ulong addr, len;
-
- /*
- * This function should never be called with addresses outside the
- * guest address space. If this assert fires, it probably indicates
- * a missing call to h2g_valid.
- */
- assert(end - 1 <= GUEST_ADDR_MAX);
- assert(start < end);
- assert_memory_lock();
-
- start = start & TARGET_PAGE_MASK;
- end = TARGET_PAGE_ALIGN(end);
-
- for (addr = start, len = end - start;
- len != 0;
- len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
- PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
-
- g_free(p->target_data);
- p->target_data = NULL;
- }
-}
-void *page_get_target_data(target_ulong address)
-{
- PageDesc *p = page_find(address >> TARGET_PAGE_BITS);
- return p ? p->target_data : NULL;
-}
-
-void *page_alloc_target_data(target_ulong address, size_t size)
-{
- PageDesc *p = page_find(address >> TARGET_PAGE_BITS);
- void *ret = NULL;
-
- if (p->flags & PAGE_VALID) {
- ret = p->target_data;
- if (!ret) {
- p->target_data = ret = g_malloc0(size);
- }
+ if (inval_tb) {
+ tb_invalidate_phys_range(start, end);
}
- return ret;
}
int page_check_range(target_ulong start, target_ulong len, int flags)
@@ -2396,9 +1488,6 @@ void page_protect(tb_page_addr_t page_addr)
}
mprotect(g2h_untagged(page_addr), qemu_host_page_size,
(prot & PAGE_BITS) & ~PAGE_WRITE);
- if (DEBUG_TB_INVALIDATE_GATE) {
- printf("protecting code page: 0x" TB_PAGE_ADDR_FMT "\n", page_addr);
- }
}
}
@@ -2453,12 +1542,8 @@ int page_unprotect(target_ulong address, uintptr_t pc)
/* and since the content will be modified, we must invalidate
the corresponding translated code. */
- current_tb_invalidated |= tb_invalidate_phys_page(addr, pc);
-#ifdef CONFIG_USER_ONLY
- if (DEBUG_TB_CHECK_GATE) {
- tb_invalidate_check(addr);
- }
-#endif
+ current_tb_invalidated |=
+ tb_invalidate_phys_page_unwind(addr, pc);
}
mprotect((void *)g2h_untagged(host_start), qemu_host_page_size,
prot & PAGE_BITS);
diff --git a/accel/tcg/translator.c b/accel/tcg/translator.c
index 8e78fd7a9c..061519691f 100644
--- a/accel/tcg/translator.c
+++ b/accel/tcg/translator.c
@@ -157,7 +157,7 @@ static void *translator_access(CPUArchState *env, DisasContextBase *db,
tb = db->tb;
/* Use slow path if first page is MMIO. */
- if (unlikely(tb->page_addr[0] == -1)) {
+ if (unlikely(tb_page_addr0(tb) == -1)) {
return NULL;
}
@@ -169,13 +169,14 @@ static void *translator_access(CPUArchState *env, DisasContextBase *db,
host = db->host_addr[1];
base = TARGET_PAGE_ALIGN(db->pc_first);
if (host == NULL) {
- tb->page_addr[1] =
+ tb_page_addr_t phys_page =
get_page_addr_code_hostp(env, base, &db->host_addr[1]);
+ /* We cannot handle MMIO as second page. */
+ assert(phys_page != -1);
+ tb_set_page_addr1(tb, phys_page);
#ifdef CONFIG_USER_ONLY
page_protect(end);
#endif
- /* We cannot handle MMIO as second page. */
- assert(tb->page_addr[1] != -1);
host = db->host_addr[1];
}
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index 521aa8b61e..fb7d6ee9e9 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -210,6 +210,48 @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr,
return addr;
}
+void page_reset_target_data(target_ulong start, target_ulong end)
+{
+#ifdef TARGET_PAGE_DATA_SIZE
+ target_ulong addr, len;
+
+ /*
+ * This function should never be called with addresses outside the
+ * guest address space. If this assert fires, it probably indicates
+ * a missing call to h2g_valid.
+ */
+ assert(end - 1 <= GUEST_ADDR_MAX);
+ assert(start < end);
+ assert_memory_lock();
+
+ start = start & TARGET_PAGE_MASK;
+ end = TARGET_PAGE_ALIGN(end);
+
+ for (addr = start, len = end - start;
+ len != 0;
+ len -= TARGET_PAGE_SIZE, addr += TARGET_PAGE_SIZE) {
+ PageDesc *p = page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
+
+ g_free(p->target_data);
+ p->target_data = NULL;
+ }
+#endif
+}
+
+#ifdef TARGET_PAGE_DATA_SIZE
+void *page_get_target_data(target_ulong address)
+{
+ PageDesc *p = page_find(address >> TARGET_PAGE_BITS);
+ void *ret = p->target_data;
+
+ if (!ret) {
+ ret = g_malloc0(TARGET_PAGE_DATA_SIZE);
+ p->target_data = ret;
+ }
+ return ret;
+}
+#endif
+
/* The softmmu versions of these helpers are in cputlb.c. */
/*
generated by cgit v1.2.3 (git 2.26.2) at 2024-12-25 14:04:13 +0000