aboutsummaryrefslogtreecommitdiff
path: root/accel
diff options
context:
space:
mode:
Diffstat (limited to 'accel')
-rw-r--r--accel/tcg/atomic_template.h8
-rw-r--r--accel/tcg/cpu-exec.c23
-rw-r--r--accel/tcg/cputlb.c226
-rw-r--r--accel/tcg/tcg-all.c6
-rw-r--r--accel/tcg/tcg-runtime.c4
-rw-r--r--accel/tcg/translate-all.c10
-rw-r--r--accel/tcg/user-exec.c2
7 files changed, 145 insertions, 134 deletions
diff --git a/accel/tcg/atomic_template.h b/accel/tcg/atomic_template.h
index 685602b076..5aaf186253 100644
--- a/accel/tcg/atomic_template.h
+++ b/accel/tcg/atomic_template.h
@@ -62,21 +62,21 @@
#define ATOMIC_TRACE_RMW do { \
uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
\
- trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
- trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, \
+ trace_guest_mem_before_exec(env_cpu(env), addr, info); \
+ trace_guest_mem_before_exec(env_cpu(env), addr, \
info | TRACE_MEM_ST); \
} while (0)
#define ATOMIC_TRACE_LD do { \
uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, false); \
\
- trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
+ trace_guest_mem_before_exec(env_cpu(env), addr, info); \
} while (0)
# define ATOMIC_TRACE_ST do { \
uint8_t info = glue(trace_mem_build_info_no_se, MEND)(SHIFT, true); \
\
- trace_guest_mem_before_exec(ENV_GET_CPU(env), addr, info); \
+ trace_guest_mem_before_exec(env_cpu(env), addr, info); \
} while (0)
/* Define host-endian atomic operations. Note that END is used within
diff --git a/accel/tcg/cpu-exec.c b/accel/tcg/cpu-exec.c
index 45ef41ebb2..032a62672e 100644
--- a/accel/tcg/cpu-exec.c
+++ b/accel/tcg/cpu-exec.c
@@ -54,7 +54,7 @@ typedef struct SyncClocks {
#define MAX_DELAY_PRINT_RATE 2000000000LL
#define MAX_NB_PRINTS 100
-static void align_clocks(SyncClocks *sc, const CPUState *cpu)
+static void align_clocks(SyncClocks *sc, CPUState *cpu)
{
int64_t cpu_icount;
@@ -62,7 +62,7 @@ static void align_clocks(SyncClocks *sc, const CPUState *cpu)
return;
}
- cpu_icount = cpu->icount_extra + cpu->icount_decr.u16.low;
+ cpu_icount = cpu->icount_extra + cpu_neg(cpu)->icount_decr.u16.low;
sc->diff_clk += cpu_icount_to_ns(sc->last_cpu_icount - cpu_icount);
sc->last_cpu_icount = cpu_icount;
@@ -105,15 +105,15 @@ static void print_delay(const SyncClocks *sc)
}
}
-static void init_delay_params(SyncClocks *sc,
- const CPUState *cpu)
+static void init_delay_params(SyncClocks *sc, CPUState *cpu)
{
if (!icount_align_option) {
return;
}
sc->realtime_clock = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL_RT);
sc->diff_clk = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) - sc->realtime_clock;
- sc->last_cpu_icount = cpu->icount_extra + cpu->icount_decr.u16.low;
+ sc->last_cpu_icount
+ = cpu->icount_extra + cpu_neg(cpu)->icount_decr.u16.low;
if (sc->diff_clk < max_delay) {
max_delay = sc->diff_clk;
}
@@ -467,7 +467,7 @@ static inline bool cpu_handle_exception(CPUState *cpu, int *ret)
if (cpu->exception_index < 0) {
#ifndef CONFIG_USER_ONLY
if (replay_has_exception()
- && cpu->icount_decr.u16.low + cpu->icount_extra == 0) {
+ && cpu_neg(cpu)->icount_decr.u16.low + cpu->icount_extra == 0) {
/* try to cause an exception pending in the log */
cpu_exec_nocache(cpu, 1, tb_find(cpu, NULL, 0, curr_cflags()), true);
}
@@ -525,7 +525,7 @@ static inline bool cpu_handle_interrupt(CPUState *cpu,
* Ensure zeroing happens before reading cpu->exit_request or
* cpu->interrupt_request (see also smp_wmb in cpu_exit())
*/
- atomic_mb_set(&cpu->icount_decr.u16.high, 0);
+ atomic_mb_set(&cpu_neg(cpu)->icount_decr.u16.high, 0);
if (unlikely(atomic_read(&cpu->interrupt_request))) {
int interrupt_request;
@@ -596,8 +596,9 @@ static inline bool cpu_handle_interrupt(CPUState *cpu,
}
/* Finally, check if we need to exit to the main loop. */
- if (unlikely(atomic_read(&cpu->exit_request)
- || (use_icount && cpu->icount_decr.u16.low + cpu->icount_extra == 0))) {
+ if (unlikely(atomic_read(&cpu->exit_request))
+ || (use_icount
+ && cpu_neg(cpu)->icount_decr.u16.low + cpu->icount_extra == 0)) {
atomic_set(&cpu->exit_request, 0);
if (cpu->exception_index == -1) {
cpu->exception_index = EXCP_INTERRUPT;
@@ -624,7 +625,7 @@ static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb,
}
*last_tb = NULL;
- insns_left = atomic_read(&cpu->icount_decr.u32);
+ insns_left = atomic_read(&cpu_neg(cpu)->icount_decr.u32);
if (insns_left < 0) {
/* Something asked us to stop executing chained TBs; just
* continue round the main loop. Whatever requested the exit
@@ -643,7 +644,7 @@ static inline void cpu_loop_exec_tb(CPUState *cpu, TranslationBlock *tb,
cpu_update_icount(cpu);
/* Refill decrementer and continue execution. */
insns_left = MIN(0xffff, cpu->icount_budget);
- cpu->icount_decr.u16.low = insns_left;
+ cpu_neg(cpu)->icount_decr.u16.low = insns_left;
cpu->icount_extra = cpu->icount_budget - insns_left;
if (!cpu->icount_extra) {
/* Execute any remaining instructions, then let the main loop
diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c
index cdcc377102..baa3eb8f92 100644
--- a/accel/tcg/cputlb.c
+++ b/accel/tcg/cputlb.c
@@ -76,14 +76,14 @@ QEMU_BUILD_BUG_ON(NB_MMU_MODES > 16);
static inline size_t sizeof_tlb(CPUArchState *env, uintptr_t mmu_idx)
{
- return env->tlb_mask[mmu_idx] + (1 << CPU_TLB_ENTRY_BITS);
+ return env_tlb(env)->f[mmu_idx].mask + (1 << CPU_TLB_ENTRY_BITS);
}
-static void tlb_window_reset(CPUTLBWindow *window, int64_t ns,
+static void tlb_window_reset(CPUTLBDesc *desc, int64_t ns,
size_t max_entries)
{
- window->begin_ns = ns;
- window->max_entries = max_entries;
+ desc->window_begin_ns = ns;
+ desc->window_max_entries = max_entries;
}
static void tlb_dyn_init(CPUArchState *env)
@@ -91,14 +91,14 @@ static void tlb_dyn_init(CPUArchState *env)
int i;
for (i = 0; i < NB_MMU_MODES; i++) {
- CPUTLBDesc *desc = &env->tlb_d[i];
+ CPUTLBDesc *desc = &env_tlb(env)->d[i];
size_t n_entries = 1 << CPU_TLB_DYN_DEFAULT_BITS;
- tlb_window_reset(&desc->window, get_clock_realtime(), 0);
+ tlb_window_reset(desc, get_clock_realtime(), 0);
desc->n_used_entries = 0;
- env->tlb_mask[i] = (n_entries - 1) << CPU_TLB_ENTRY_BITS;
- env->tlb_table[i] = g_new(CPUTLBEntry, n_entries);
- env->iotlb[i] = g_new(CPUIOTLBEntry, n_entries);
+ env_tlb(env)->f[i].mask = (n_entries - 1) << CPU_TLB_ENTRY_BITS;
+ env_tlb(env)->f[i].table = g_new(CPUTLBEntry, n_entries);
+ env_tlb(env)->d[i].iotlb = g_new(CPUIOTLBEntry, n_entries);
}
}
@@ -144,25 +144,25 @@ static void tlb_dyn_init(CPUArchState *env)
*/
static void tlb_mmu_resize_locked(CPUArchState *env, int mmu_idx)
{
- CPUTLBDesc *desc = &env->tlb_d[mmu_idx];
+ CPUTLBDesc *desc = &env_tlb(env)->d[mmu_idx];
size_t old_size = tlb_n_entries(env, mmu_idx);
size_t rate;
size_t new_size = old_size;
int64_t now = get_clock_realtime();
int64_t window_len_ms = 100;
int64_t window_len_ns = window_len_ms * 1000 * 1000;
- bool window_expired = now > desc->window.begin_ns + window_len_ns;
+ bool window_expired = now > desc->window_begin_ns + window_len_ns;
- if (desc->n_used_entries > desc->window.max_entries) {
- desc->window.max_entries = desc->n_used_entries;
+ if (desc->n_used_entries > desc->window_max_entries) {
+ desc->window_max_entries = desc->n_used_entries;
}
- rate = desc->window.max_entries * 100 / old_size;
+ rate = desc->window_max_entries * 100 / old_size;
if (rate > 70) {
new_size = MIN(old_size << 1, 1 << CPU_TLB_DYN_MAX_BITS);
} else if (rate < 30 && window_expired) {
- size_t ceil = pow2ceil(desc->window.max_entries);
- size_t expected_rate = desc->window.max_entries * 100 / ceil;
+ size_t ceil = pow2ceil(desc->window_max_entries);
+ size_t expected_rate = desc->window_max_entries * 100 / ceil;
/*
* Avoid undersizing when the max number of entries seen is just below
@@ -182,19 +182,19 @@ static void tlb_mmu_resize_locked(CPUArchState *env, int mmu_idx)
if (new_size == old_size) {
if (window_expired) {
- tlb_window_reset(&desc->window, now, desc->n_used_entries);
+ tlb_window_reset(desc, now, desc->n_used_entries);
}
return;
}
- g_free(env->tlb_table[mmu_idx]);
- g_free(env->iotlb[mmu_idx]);
+ g_free(env_tlb(env)->f[mmu_idx].table);
+ g_free(env_tlb(env)->d[mmu_idx].iotlb);
- tlb_window_reset(&desc->window, now, 0);
+ tlb_window_reset(desc, now, 0);
/* desc->n_used_entries is cleared by the caller */
- env->tlb_mask[mmu_idx] = (new_size - 1) << CPU_TLB_ENTRY_BITS;
- env->tlb_table[mmu_idx] = g_try_new(CPUTLBEntry, new_size);
- env->iotlb[mmu_idx] = g_try_new(CPUIOTLBEntry, new_size);
+ env_tlb(env)->f[mmu_idx].mask = (new_size - 1) << CPU_TLB_ENTRY_BITS;
+ env_tlb(env)->f[mmu_idx].table = g_try_new(CPUTLBEntry, new_size);
+ env_tlb(env)->d[mmu_idx].iotlb = g_try_new(CPUIOTLBEntry, new_size);
/*
* If the allocations fail, try smaller sizes. We just freed some
* memory, so going back to half of new_size has a good chance of working.
@@ -202,46 +202,47 @@ static void tlb_mmu_resize_locked(CPUArchState *env, int mmu_idx)
* allocations to fail though, so we progressively reduce the allocation
* size, aborting if we cannot even allocate the smallest TLB we support.
*/
- while (env->tlb_table[mmu_idx] == NULL || env->iotlb[mmu_idx] == NULL) {
+ while (env_tlb(env)->f[mmu_idx].table == NULL ||
+ env_tlb(env)->d[mmu_idx].iotlb == NULL) {
if (new_size == (1 << CPU_TLB_DYN_MIN_BITS)) {
error_report("%s: %s", __func__, strerror(errno));
abort();
}
new_size = MAX(new_size >> 1, 1 << CPU_TLB_DYN_MIN_BITS);
- env->tlb_mask[mmu_idx] = (new_size - 1) << CPU_TLB_ENTRY_BITS;
+ env_tlb(env)->f[mmu_idx].mask = (new_size - 1) << CPU_TLB_ENTRY_BITS;
- g_free(env->tlb_table[mmu_idx]);
- g_free(env->iotlb[mmu_idx]);
- env->tlb_table[mmu_idx] = g_try_new(CPUTLBEntry, new_size);
- env->iotlb[mmu_idx] = g_try_new(CPUIOTLBEntry, new_size);
+ g_free(env_tlb(env)->f[mmu_idx].table);
+ g_free(env_tlb(env)->d[mmu_idx].iotlb);
+ env_tlb(env)->f[mmu_idx].table = g_try_new(CPUTLBEntry, new_size);
+ env_tlb(env)->d[mmu_idx].iotlb = g_try_new(CPUIOTLBEntry, new_size);
}
}
static inline void tlb_table_flush_by_mmuidx(CPUArchState *env, int mmu_idx)
{
tlb_mmu_resize_locked(env, mmu_idx);
- memset(env->tlb_table[mmu_idx], -1, sizeof_tlb(env, mmu_idx));
- env->tlb_d[mmu_idx].n_used_entries = 0;
+ memset(env_tlb(env)->f[mmu_idx].table, -1, sizeof_tlb(env, mmu_idx));
+ env_tlb(env)->d[mmu_idx].n_used_entries = 0;
}
static inline void tlb_n_used_entries_inc(CPUArchState *env, uintptr_t mmu_idx)
{
- env->tlb_d[mmu_idx].n_used_entries++;
+ env_tlb(env)->d[mmu_idx].n_used_entries++;
}
static inline void tlb_n_used_entries_dec(CPUArchState *env, uintptr_t mmu_idx)
{
- env->tlb_d[mmu_idx].n_used_entries--;
+ env_tlb(env)->d[mmu_idx].n_used_entries--;
}
void tlb_init(CPUState *cpu)
{
CPUArchState *env = cpu->env_ptr;
- qemu_spin_init(&env->tlb_c.lock);
+ qemu_spin_init(&env_tlb(env)->c.lock);
/* Ensure that cpu_reset performs a full flush. */
- env->tlb_c.dirty = ALL_MMUIDX_BITS;
+ env_tlb(env)->c.dirty = ALL_MMUIDX_BITS;
tlb_dyn_init(env);
}
@@ -273,9 +274,9 @@ void tlb_flush_counts(size_t *pfull, size_t *ppart, size_t *pelide)
CPU_FOREACH(cpu) {
CPUArchState *env = cpu->env_ptr;
- full += atomic_read(&env->tlb_c.full_flush_count);
- part += atomic_read(&env->tlb_c.part_flush_count);
- elide += atomic_read(&env->tlb_c.elide_flush_count);
+ full += atomic_read(&env_tlb(env)->c.full_flush_count);
+ part += atomic_read(&env_tlb(env)->c.part_flush_count);
+ elide += atomic_read(&env_tlb(env)->c.elide_flush_count);
}
*pfull = full;
*ppart = part;
@@ -285,10 +286,11 @@ void tlb_flush_counts(size_t *pfull, size_t *ppart, size_t *pelide)
static void tlb_flush_one_mmuidx_locked(CPUArchState *env, int mmu_idx)
{
tlb_table_flush_by_mmuidx(env, mmu_idx);
- memset(env->tlb_v_table[mmu_idx], -1, sizeof(env->tlb_v_table[0]));
- env->tlb_d[mmu_idx].large_page_addr = -1;
- env->tlb_d[mmu_idx].large_page_mask = -1;
- env->tlb_d[mmu_idx].vindex = 0;
+ env_tlb(env)->d[mmu_idx].large_page_addr = -1;
+ env_tlb(env)->d[mmu_idx].large_page_mask = -1;
+ env_tlb(env)->d[mmu_idx].vindex = 0;
+ memset(env_tlb(env)->d[mmu_idx].vtable, -1,
+ sizeof(env_tlb(env)->d[0].vtable));
}
static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
@@ -301,31 +303,31 @@ static void tlb_flush_by_mmuidx_async_work(CPUState *cpu, run_on_cpu_data data)
tlb_debug("mmu_idx:0x%04" PRIx16 "\n", asked);
- qemu_spin_lock(&env->tlb_c.lock);
+ qemu_spin_lock(&env_tlb(env)->c.lock);
- all_dirty = env->tlb_c.dirty;
+ all_dirty = env_tlb(env)->c.dirty;
to_clean = asked & all_dirty;
all_dirty &= ~to_clean;
- env->tlb_c.dirty = all_dirty;
+ env_tlb(env)->c.dirty = all_dirty;
for (work = to_clean; work != 0; work &= work - 1) {
int mmu_idx = ctz32(work);
tlb_flush_one_mmuidx_locked(env, mmu_idx);
}
- qemu_spin_unlock(&env->tlb_c.lock);
+ qemu_spin_unlock(&env_tlb(env)->c.lock);
cpu_tb_jmp_cache_clear(cpu);
if (to_clean == ALL_MMUIDX_BITS) {
- atomic_set(&env->tlb_c.full_flush_count,
- env->tlb_c.full_flush_count + 1);
+ atomic_set(&env_tlb(env)->c.full_flush_count,
+ env_tlb(env)->c.full_flush_count + 1);
} else {
- atomic_set(&env->tlb_c.part_flush_count,
- env->tlb_c.part_flush_count + ctpop16(to_clean));
+ atomic_set(&env_tlb(env)->c.part_flush_count,
+ env_tlb(env)->c.part_flush_count + ctpop16(to_clean));
if (to_clean != asked) {
- atomic_set(&env->tlb_c.elide_flush_count,
- env->tlb_c.elide_flush_count +
+ atomic_set(&env_tlb(env)->c.elide_flush_count,
+ env_tlb(env)->c.elide_flush_count +
ctpop16(asked & ~to_clean));
}
}
@@ -410,11 +412,12 @@ static inline bool tlb_flush_entry_locked(CPUTLBEntry *tlb_entry,
static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx,
target_ulong page)
{
+ CPUTLBDesc *d = &env_tlb(env)->d[mmu_idx];
int k;
- assert_cpu_is_self(ENV_GET_CPU(env));
+ assert_cpu_is_self(env_cpu(env));
for (k = 0; k < CPU_VTLB_SIZE; k++) {
- if (tlb_flush_entry_locked(&env->tlb_v_table[mmu_idx][k], page)) {
+ if (tlb_flush_entry_locked(&d->vtable[k], page)) {
tlb_n_used_entries_dec(env, mmu_idx);
}
}
@@ -423,8 +426,8 @@ static inline void tlb_flush_vtlb_page_locked(CPUArchState *env, int mmu_idx,
static void tlb_flush_page_locked(CPUArchState *env, int midx,
target_ulong page)
{
- target_ulong lp_addr = env->tlb_d[midx].large_page_addr;
- target_ulong lp_mask = env->tlb_d[midx].large_page_mask;
+ target_ulong lp_addr = env_tlb(env)->d[midx].large_page_addr;
+ target_ulong lp_mask = env_tlb(env)->d[midx].large_page_mask;
/* Check if we need to flush due to large pages. */
if ((page & lp_mask) == lp_addr) {
@@ -459,13 +462,13 @@ static void tlb_flush_page_by_mmuidx_async_work(CPUState *cpu,
tlb_debug("page addr:" TARGET_FMT_lx " mmu_map:0x%lx\n",
addr, mmu_idx_bitmap);
- qemu_spin_lock(&env->tlb_c.lock);
+ qemu_spin_lock(&env_tlb(env)->c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
if (test_bit(mmu_idx, &mmu_idx_bitmap)) {
tlb_flush_page_locked(env, mmu_idx, addr);
}
}
- qemu_spin_unlock(&env->tlb_c.lock);
+ qemu_spin_unlock(&env_tlb(env)->c.lock);
tb_flush_jmp_cache(cpu, addr);
}
@@ -609,22 +612,22 @@ void tlb_reset_dirty(CPUState *cpu, ram_addr_t start1, ram_addr_t length)
int mmu_idx;
env = cpu->env_ptr;
- qemu_spin_lock(&env->tlb_c.lock);
+ qemu_spin_lock(&env_tlb(env)->c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
unsigned int i;
unsigned int n = tlb_n_entries(env, mmu_idx);
for (i = 0; i < n; i++) {
- tlb_reset_dirty_range_locked(&env->tlb_table[mmu_idx][i], start1,
- length);
+ tlb_reset_dirty_range_locked(&env_tlb(env)->f[mmu_idx].table[i],
+ start1, length);
}
for (i = 0; i < CPU_VTLB_SIZE; i++) {
- tlb_reset_dirty_range_locked(&env->tlb_v_table[mmu_idx][i], start1,
- length);
+ tlb_reset_dirty_range_locked(&env_tlb(env)->d[mmu_idx].vtable[i],
+ start1, length);
}
}
- qemu_spin_unlock(&env->tlb_c.lock);
+ qemu_spin_unlock(&env_tlb(env)->c.lock);
}
/* Called with tlb_c.lock held */
@@ -646,7 +649,7 @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
assert_cpu_is_self(cpu);
vaddr &= TARGET_PAGE_MASK;
- qemu_spin_lock(&env->tlb_c.lock);
+ qemu_spin_lock(&env_tlb(env)->c.lock);
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
tlb_set_dirty1_locked(tlb_entry(env, mmu_idx, vaddr), vaddr);
}
@@ -654,10 +657,10 @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
for (mmu_idx = 0; mmu_idx < NB_MMU_MODES; mmu_idx++) {
int k;
for (k = 0; k < CPU_VTLB_SIZE; k++) {
- tlb_set_dirty1_locked(&env->tlb_v_table[mmu_idx][k], vaddr);
+ tlb_set_dirty1_locked(&env_tlb(env)->d[mmu_idx].vtable[k], vaddr);
}
}
- qemu_spin_unlock(&env->tlb_c.lock);
+ qemu_spin_unlock(&env_tlb(env)->c.lock);
}
/* Our TLB does not support large pages, so remember the area covered by
@@ -665,7 +668,7 @@ void tlb_set_dirty(CPUState *cpu, target_ulong vaddr)
static void tlb_add_large_page(CPUArchState *env, int mmu_idx,
target_ulong vaddr, target_ulong size)
{
- target_ulong lp_addr = env->tlb_d[mmu_idx].large_page_addr;
+ target_ulong lp_addr = env_tlb(env)->d[mmu_idx].large_page_addr;
target_ulong lp_mask = ~(size - 1);
if (lp_addr == (target_ulong)-1) {
@@ -675,13 +678,13 @@ static void tlb_add_large_page(CPUArchState *env, int mmu_idx,
/* Extend the existing region to include the new page.
This is a compromise between unnecessary flushes and
the cost of maintaining a full variable size TLB. */
- lp_mask &= env->tlb_d[mmu_idx].large_page_mask;
+ lp_mask &= env_tlb(env)->d[mmu_idx].large_page_mask;
while (((lp_addr ^ vaddr) & lp_mask) != 0) {
lp_mask <<= 1;
}
}
- env->tlb_d[mmu_idx].large_page_addr = lp_addr & lp_mask;
- env->tlb_d[mmu_idx].large_page_mask = lp_mask;
+ env_tlb(env)->d[mmu_idx].large_page_addr = lp_addr & lp_mask;
+ env_tlb(env)->d[mmu_idx].large_page_mask = lp_mask;
}
/* Add a new TLB entry. At most one entry for a given virtual address
@@ -696,6 +699,8 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
int mmu_idx, target_ulong size)
{
CPUArchState *env = cpu->env_ptr;
+ CPUTLB *tlb = env_tlb(env);
+ CPUTLBDesc *desc = &tlb->d[mmu_idx];
MemoryRegionSection *section;
unsigned int index;
target_ulong address;
@@ -757,10 +762,10 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
* a longer critical section, but this is not a concern since the TLB lock
* is unlikely to be contended.
*/
- qemu_spin_lock(&env->tlb_c.lock);
+ qemu_spin_lock(&tlb->c.lock);
/* Note that the tlb is no longer clean. */
- env->tlb_c.dirty |= 1 << mmu_idx;
+ tlb->c.dirty |= 1 << mmu_idx;
/* Make sure there's no cached translation for the new page. */
tlb_flush_vtlb_page_locked(env, mmu_idx, vaddr_page);
@@ -770,12 +775,12 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
* different page; otherwise just overwrite the stale data.
*/
if (!tlb_hit_page_anyprot(te, vaddr_page) && !tlb_entry_is_empty(te)) {
- unsigned vidx = env->tlb_d[mmu_idx].vindex++ % CPU_VTLB_SIZE;
- CPUTLBEntry *tv = &env->tlb_v_table[mmu_idx][vidx];
+ unsigned vidx = desc->vindex++ % CPU_VTLB_SIZE;
+ CPUTLBEntry *tv = &desc->vtable[vidx];
/* Evict the old entry into the victim tlb. */
copy_tlb_helper_locked(tv, te);
- env->iotlb_v[mmu_idx][vidx] = env->iotlb[mmu_idx][index];
+ desc->viotlb[vidx] = desc->iotlb[index];
tlb_n_used_entries_dec(env, mmu_idx);
}
@@ -792,8 +797,8 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
* subtract here is that of the page base, and not the same as the
* vaddr we add back in io_readx()/io_writex()/get_page_addr_code().
*/
- env->iotlb[mmu_idx][index].addr = iotlb - vaddr_page;
- env->iotlb[mmu_idx][index].attrs = attrs;
+ desc->iotlb[index].addr = iotlb - vaddr_page;
+ desc->iotlb[index].attrs = attrs;
/* Now calculate the new entry */
tn.addend = addend - vaddr_page;
@@ -829,7 +834,7 @@ void tlb_set_page_with_attrs(CPUState *cpu, target_ulong vaddr,
copy_tlb_helper_locked(te, &tn);
tlb_n_used_entries_inc(env, mmu_idx);
- qemu_spin_unlock(&env->tlb_c.lock);
+ qemu_spin_unlock(&tlb->c.lock);
}
/* Add a new TLB entry, but without specifying the memory
@@ -878,7 +883,7 @@ static uint64_t io_readx(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
int mmu_idx, target_ulong addr, uintptr_t retaddr,
MMUAccessType access_type, int size)
{
- CPUState *cpu = ENV_GET_CPU(env);
+ CPUState *cpu = env_cpu(env);
hwaddr mr_offset;
MemoryRegionSection *section;
MemoryRegion *mr;
@@ -922,7 +927,7 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry,
int mmu_idx, uint64_t val, target_ulong addr,
uintptr_t retaddr, int size)
{
- CPUState *cpu = ENV_GET_CPU(env);
+ CPUState *cpu = env_cpu(env);
hwaddr mr_offset;
MemoryRegionSection *section;
MemoryRegion *mr;
@@ -974,23 +979,30 @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index,
{
size_t vidx;
- assert_cpu_is_self(ENV_GET_CPU(env));
+ assert_cpu_is_self(env_cpu(env));
for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) {
- CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx];
- target_ulong cmp = tlb_read_ofs(vtlb, elt_ofs);
+ CPUTLBEntry *vtlb = &env_tlb(env)->d[mmu_idx].vtable[vidx];
+ target_ulong cmp;
+
+ /* elt_ofs might correspond to .addr_write, so use atomic_read */
+#if TCG_OVERSIZED_GUEST
+ cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs);
+#else
+ cmp = atomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs));
+#endif
if (cmp == page) {
/* Found entry in victim tlb, swap tlb and iotlb. */
- CPUTLBEntry tmptlb, *tlb = &env->tlb_table[mmu_idx][index];
+ CPUTLBEntry tmptlb, *tlb = &env_tlb(env)->f[mmu_idx].table[index];
- qemu_spin_lock(&env->tlb_c.lock);
+ qemu_spin_lock(&env_tlb(env)->c.lock);
copy_tlb_helper_locked(&tmptlb, tlb);
copy_tlb_helper_locked(tlb, vtlb);
copy_tlb_helper_locked(vtlb, &tmptlb);
- qemu_spin_unlock(&env->tlb_c.lock);
+ qemu_spin_unlock(&env_tlb(env)->c.lock);
- CPUIOTLBEntry tmpio, *io = &env->iotlb[mmu_idx][index];
- CPUIOTLBEntry *vio = &env->iotlb_v[mmu_idx][vidx];
+ CPUIOTLBEntry tmpio, *io = &env_tlb(env)->d[mmu_idx].iotlb[index];
+ CPUIOTLBEntry *vio = &env_tlb(env)->d[mmu_idx].viotlb[vidx];
tmpio = *io; *io = *vio; *vio = tmpio;
return true;
}
@@ -1017,7 +1029,7 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env, target_ulong addr)
if (unlikely(!tlb_hit(entry->addr_code, addr))) {
if (!VICTIM_TLB_HIT(addr_code, addr)) {
- tlb_fill(ENV_GET_CPU(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0);
+ tlb_fill(env_cpu(env), addr, 0, MMU_INST_FETCH, mmu_idx, 0);
index = tlb_index(env, mmu_idx, addr);
entry = tlb_entry(env, mmu_idx, addr);
}
@@ -1055,7 +1067,7 @@ void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx,
if (!tlb_hit(tlb_addr_write(entry), addr)) {
/* TLB entry is for a different page */
if (!VICTIM_TLB_HIT(addr_write, addr)) {
- tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE,
+ tlb_fill(env_cpu(env), addr, size, MMU_DATA_STORE,
mmu_idx, retaddr);
}
}
@@ -1089,7 +1101,7 @@ void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr,
uintptr_t index = tlb_index(env, mmu_idx, addr);
if (!victim_tlb_hit(env, mmu_idx, index, elt_ofs, page)) {
- CPUState *cs = ENV_GET_CPU(env);
+ CPUState *cs = env_cpu(env);
CPUClass *cc = CPU_GET_CLASS(cs);
if (!cc->tlb_fill(cs, addr, 0, access_type, mmu_idx, true, 0)) {
@@ -1132,7 +1144,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
/* Enforce guest required alignment. */
if (unlikely(a_bits > 0 && (addr & ((1 << a_bits) - 1)))) {
/* ??? Maybe indicate atomic op to cpu_unaligned_access */
- cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
+ cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
mmu_idx, retaddr);
}
@@ -1148,7 +1160,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
/* Check TLB entry and enforce page permissions. */
if (!tlb_hit(tlb_addr, addr)) {
if (!VICTIM_TLB_HIT(addr_write, addr)) {
- tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_STORE,
+ tlb_fill(env_cpu(env), addr, 1 << s_bits, MMU_DATA_STORE,
mmu_idx, retaddr);
index = tlb_index(env, mmu_idx, addr);
tlbe = tlb_entry(env, mmu_idx, addr);
@@ -1165,7 +1177,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
/* Let the guest notice RMW on a write-only page. */
if (unlikely(tlbe->addr_read != (tlb_addr & ~TLB_NOTDIRTY))) {
- tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_LOAD,
+ tlb_fill(env_cpu(env), addr, 1 << s_bits, MMU_DATA_LOAD,
mmu_idx, retaddr);
/* Since we don't support reads and writes to different addresses,
and we do have the proper page loaded for write, this shouldn't
@@ -1178,7 +1190,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
ndi->active = false;
if (unlikely(tlb_addr & TLB_NOTDIRTY)) {
ndi->active = true;
- memory_notdirty_write_prepare(ndi, ENV_GET_CPU(env), addr,
+ memory_notdirty_write_prepare(ndi, env_cpu(env), addr,
qemu_ram_addr_from_host_nofail(hostaddr),
1 << s_bits);
}
@@ -1186,7 +1198,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
return hostaddr;
stop_the_world:
- cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
+ cpu_loop_exit_atomic(env_cpu(env), retaddr);
}
#ifdef TARGET_WORDS_BIGENDIAN
@@ -1251,7 +1263,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
/* Handle CPU specific unaligned behaviour */
if (addr & ((1 << a_bits) - 1)) {
- cpu_unaligned_access(ENV_GET_CPU(env), addr, access_type,
+ cpu_unaligned_access(env_cpu(env), addr, access_type,
mmu_idx, retaddr);
}
@@ -1259,7 +1271,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
if (!tlb_hit(tlb_addr, addr)) {
if (!victim_tlb_hit(env, mmu_idx, index, tlb_off,
addr & TARGET_PAGE_MASK)) {
- tlb_fill(ENV_GET_CPU(env), addr, size,
+ tlb_fill(env_cpu(env), addr, size,
access_type, mmu_idx, retaddr);
index = tlb_index(env, mmu_idx, addr);
entry = tlb_entry(env, mmu_idx, addr);
@@ -1280,7 +1292,7 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
* repeat the MMU check here. This tlb_fill() call might
* longjump out if this access should cause a guest exception.
*/
- tlb_fill(ENV_GET_CPU(env), addr, size,
+ tlb_fill(env_cpu(env), addr, size,
access_type, mmu_idx, retaddr);
index = tlb_index(env, mmu_idx, addr);
entry = tlb_entry(env, mmu_idx, addr);
@@ -1293,8 +1305,8 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
}
}
- res = io_readx(env, &env->iotlb[mmu_idx][index], mmu_idx, addr,
- retaddr, access_type, size);
+ res = io_readx(env, &env_tlb(env)->d[mmu_idx].iotlb[index],
+ mmu_idx, addr, retaddr, access_type, size);
return handle_bswap(res, size, big_endian);
}
@@ -1499,7 +1511,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
/* Handle CPU specific unaligned behaviour */
if (addr & ((1 << a_bits) - 1)) {
- cpu_unaligned_access(ENV_GET_CPU(env), addr, MMU_DATA_STORE,
+ cpu_unaligned_access(env_cpu(env), addr, MMU_DATA_STORE,
mmu_idx, retaddr);
}
@@ -1507,7 +1519,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
if (!tlb_hit(tlb_addr, addr)) {
if (!victim_tlb_hit(env, mmu_idx, index, tlb_off,
addr & TARGET_PAGE_MASK)) {
- tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE,
+ tlb_fill(env_cpu(env), addr, size, MMU_DATA_STORE,
mmu_idx, retaddr);
index = tlb_index(env, mmu_idx, addr);
entry = tlb_entry(env, mmu_idx, addr);
@@ -1528,7 +1540,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
* repeat the MMU check here. This tlb_fill() call might
* longjump out if this access should cause a guest exception.
*/
- tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE,
+ tlb_fill(env_cpu(env), addr, size, MMU_DATA_STORE,
mmu_idx, retaddr);
index = tlb_index(env, mmu_idx, addr);
entry = tlb_entry(env, mmu_idx, addr);
@@ -1541,7 +1553,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
}
}
- io_writex(env, &env->iotlb[mmu_idx][index], mmu_idx,
+ io_writex(env, &env_tlb(env)->d[mmu_idx].iotlb[index], mmu_idx,
handle_bswap(val, size, big_endian),
addr, retaddr, size);
return;
@@ -1568,7 +1580,7 @@ store_helper(CPUArchState *env, target_ulong addr, uint64_t val,
if (!tlb_hit_page(tlb_addr2, page2)
&& !victim_tlb_hit(env, mmu_idx, index2, tlb_off,
page2 & TARGET_PAGE_MASK)) {
- tlb_fill(ENV_GET_CPU(env), page2, size, MMU_DATA_STORE,
+ tlb_fill(env_cpu(env), page2, size, MMU_DATA_STORE,
mmu_idx, retaddr);
}
diff --git a/accel/tcg/tcg-all.c b/accel/tcg/tcg-all.c
index 3d25bdcc17..9b215dcc5a 100644
--- a/accel/tcg/tcg-all.c
+++ b/accel/tcg/tcg-all.c
@@ -28,13 +28,12 @@
#include "sysemu/sysemu.h"
#include "qom/object.h"
#include "qemu-common.h"
-#include "qom/cpu.h"
+#include "cpu.h"
#include "sysemu/cpus.h"
#include "qemu/main-loop.h"
unsigned long tcg_tb_size;
-#ifndef CONFIG_USER_ONLY
/* mask must never be zero, except for A20 change call */
static void tcg_handle_interrupt(CPUState *cpu, int mask)
{
@@ -51,7 +50,7 @@ static void tcg_handle_interrupt(CPUState *cpu, int mask)
if (!qemu_cpu_is_self(cpu)) {
qemu_cpu_kick(cpu);
} else {
- atomic_set(&cpu->icount_decr.u16.high, -1);
+ atomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
if (use_icount &&
!cpu->can_do_io
&& (mask & ~old_mask) != 0) {
@@ -59,7 +58,6 @@ static void tcg_handle_interrupt(CPUState *cpu, int mask)
}
}
}
-#endif
static int tcg_init(MachineState *ms)
{
diff --git a/accel/tcg/tcg-runtime.c b/accel/tcg/tcg-runtime.c
index d0d4484406..8a1e408e31 100644
--- a/accel/tcg/tcg-runtime.c
+++ b/accel/tcg/tcg-runtime.c
@@ -146,7 +146,7 @@ uint64_t HELPER(ctpop_i64)(uint64_t arg)
void *HELPER(lookup_tb_ptr)(CPUArchState *env)
{
- CPUState *cpu = ENV_GET_CPU(env);
+ CPUState *cpu = env_cpu(env);
TranslationBlock *tb;
target_ulong cs_base, pc;
uint32_t flags;
@@ -165,5 +165,5 @@ void *HELPER(lookup_tb_ptr)(CPUArchState *env)
void HELPER(exit_atomic)(CPUArchState *env)
{
- cpu_loop_exit_atomic(ENV_GET_CPU(env), GETPC());
+ cpu_loop_exit_atomic(env_cpu(env), GETPC());
}
diff --git a/accel/tcg/translate-all.c b/accel/tcg/translate-all.c
index 20b59f93f4..e24ee3a172 100644
--- a/accel/tcg/translate-all.c
+++ b/accel/tcg/translate-all.c
@@ -364,7 +364,7 @@ static int cpu_restore_state_from_tb(CPUState *cpu, TranslationBlock *tb,
assert(use_icount);
/* Reset the cycle counter to the start of the block
and shift if to the number of actually executed instructions */
- cpu->icount_decr.u16.low += num_insns - i;
+ cpu_neg(cpu)->icount_decr.u16.low += num_insns - i;
}
restore_state_to_opc(env, tb, data);
@@ -1732,7 +1732,7 @@ TranslationBlock *tb_gen_code(CPUState *cpu,
tcg_func_start(tcg_ctx);
- tcg_ctx->cpu = ENV_GET_CPU(env);
+ tcg_ctx->cpu = env_cpu(env);
gen_intermediate_code(cpu, tb, max_insns);
tcg_ctx->cpu = NULL;
@@ -2200,7 +2200,7 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
if ((env->hflags & MIPS_HFLAG_BMASK) != 0
&& env->active_tc.PC != tb->pc) {
env->active_tc.PC -= (env->hflags & MIPS_HFLAG_B16 ? 2 : 4);
- cpu->icount_decr.u16.low++;
+ cpu_neg(cpu)->icount_decr.u16.low++;
env->hflags &= ~MIPS_HFLAG_BMASK;
n = 2;
}
@@ -2208,7 +2208,7 @@ void cpu_io_recompile(CPUState *cpu, uintptr_t retaddr)
if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
&& env->pc != tb->pc) {
env->pc -= 2;
- cpu->icount_decr.u16.low++;
+ cpu_neg(cpu)->icount_decr.u16.low++;
env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
n = 2;
}
@@ -2382,7 +2382,7 @@ void cpu_interrupt(CPUState *cpu, int mask)
{
g_assert(qemu_mutex_iothread_locked());
cpu->interrupt_request |= mask;
- atomic_set(&cpu->icount_decr.u16.high, -1);
+ atomic_set(&cpu_neg(cpu)->icount_decr.u16.high, -1);
}
/*
diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c
index 8cfbeb1b56..cb5f4b19c5 100644
--- a/accel/tcg/user-exec.c
+++ b/accel/tcg/user-exec.c
@@ -680,7 +680,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr,
{
/* Enforce qemu required alignment. */
if (unlikely(addr & (size - 1))) {
- cpu_loop_exit_atomic(ENV_GET_CPU(env), retaddr);
+ cpu_loop_exit_atomic(env_cpu(env), retaddr);
}
helper_retaddr = retaddr;
return g2h(addr);
generated by cgit v1.2.3 (git 2.26.2) at 2024-12-27 16:52:02 +0000