diff options
| -rw-r--r-- | cpu-exec.c | 1 | ||||
| -rw-r--r-- | cpus.c | 19 | ||||
| -rw-r--r-- | exec.c | 83 | ||||
| -rw-r--r-- | hw/acpi/core.c | 1 | ||||
| -rw-r--r-- | include/exec/cpu-all.h | 6 | ||||
| -rw-r--r-- | include/exec/cpu-common.h | 2 | ||||
| -rw-r--r-- | include/exec/cpu-defs.h | 15 | ||||
| -rw-r--r-- | include/exec/exec-all.h | 20 | ||||
| -rw-r--r-- | include/exec/memory.h | 27 | ||||
| -rw-r--r-- | include/exec/tb-hash.h | 51 | ||||
| -rw-r--r-- | include/hw/arm/arm.h | 1 | ||||
| -rw-r--r-- | include/qemu-common.h | 7 | ||||
| -rw-r--r-- | include/qemu/fprintf-fn.h | 17 | ||||
| -rw-r--r-- | include/qemu/main-loop.h | 10 | ||||
| -rw-r--r-- | kvm-all.c | 25 | ||||
| -rw-r--r-- | memory.c | 23 | ||||
| -rw-r--r-- | memory_mapping.c | 2 | ||||
| -rw-r--r-- | stubs/iothread-lock.c | 5 | ||||
| -rw-r--r-- | target-i386/kvm.c | 24 | ||||
| -rw-r--r-- | target-mips/kvm.c | 4 | ||||
| -rw-r--r-- | target-ppc/kvm.c | 7 | ||||
| -rw-r--r-- | target-s390x/kvm.c | 3 | ||||
| -rw-r--r-- | target-s390x/mmu_helper.c | 2 | ||||
| -rw-r--r-- | tcg/tcg.h | 2 | ||||
| -rw-r--r-- | translate-all.c | 1 | ||||
| -rw-r--r-- | util/cutils.c | 14 |
26 files changed, 299 insertions, 73 deletions
diff --git a/cpu-exec.c b/cpu-exec.c index 2ffeb6e40d..b2724c18c1 100644 --- a/cpu-exec.c +++ b/cpu-exec.c @@ -27,6 +27,7 @@ #include "exec/address-spaces.h" #include "exec/memory-internal.h" #include "qemu/rcu.h" +#include "exec/tb-hash.h" /* -icount align implementation. */ @@ -954,7 +954,7 @@ static void *qemu_kvm_cpu_thread_fn(void *arg) CPUState *cpu = arg; int r; - qemu_mutex_lock(&qemu_global_mutex); + qemu_mutex_lock_iothread(); qemu_thread_get_self(cpu->thread); cpu->thread_id = qemu_get_thread_id(); cpu->can_do_io = 1; @@ -1034,10 +1034,10 @@ static void *qemu_tcg_cpu_thread_fn(void *arg) { CPUState *cpu = arg; + qemu_mutex_lock_iothread(); qemu_tcg_init_cpu_signals(); qemu_thread_get_self(cpu->thread); - qemu_mutex_lock(&qemu_global_mutex); CPU_FOREACH(cpu) { cpu->thread_id = qemu_get_thread_id(); cpu->created = true; @@ -1146,10 +1146,21 @@ bool qemu_in_vcpu_thread(void) return current_cpu && qemu_cpu_is_self(current_cpu); } +static __thread bool iothread_locked = false; + +bool qemu_mutex_iothread_locked(void) +{ + return iothread_locked; +} + void qemu_mutex_lock_iothread(void) { atomic_inc(&iothread_requesting_mutex); - if (!tcg_enabled() || !first_cpu || !first_cpu->thread) { + /* In the simple case there is no need to bump the VCPU thread out of + * TCG code execution. + */ + if (!tcg_enabled() || qemu_in_vcpu_thread() || + !first_cpu || !first_cpu->thread) { qemu_mutex_lock(&qemu_global_mutex); atomic_dec(&iothread_requesting_mutex); } else { @@ -1160,10 +1171,12 @@ void qemu_mutex_lock_iothread(void) atomic_dec(&iothread_requesting_mutex); qemu_cond_broadcast(&qemu_io_proceeded_cond); } + iothread_locked = true; } void qemu_mutex_unlock_iothread(void) { + iothread_locked = false; qemu_mutex_unlock(&qemu_global_mutex); } @@ -48,6 +48,7 @@ #endif #include "exec/cpu-all.h" #include "qemu/rcu_queue.h" +#include "qemu/main-loop.h" #include "exec/cputlb.h" #include "translate-all.h" @@ -352,6 +353,18 @@ address_space_translate_internal(AddressSpaceDispatch *d, hwaddr addr, hwaddr *x *xlat = addr + section->offset_within_region; mr = section->mr; + + /* MMIO registers can be expected to perform full-width accesses based only + * on their address, without considering adjacent registers that could + * decode to completely different MemoryRegions. When such registers + * exist (e.g. I/O ports 0xcf8 and 0xcf9 on most PC chipsets), MMIO + * regions overlap wildly. For this reason we cannot clamp the accesses + * here. + * + * If the length is small (as is the case for address_space_ldl/stl), + * everything works fine. If the incoming length is large, however, + * the caller really has to do the clamping through memory_access_size. + */ if (memory_region_is_ram(mr)) { diff = int128_sub(section->size, int128_make64(addr)); *plen = int128_get64(int128_min(diff, int128_make64(*plen))); @@ -2316,6 +2329,29 @@ static int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr) return l; } +static bool prepare_mmio_access(MemoryRegion *mr) +{ + bool unlocked = !qemu_mutex_iothread_locked(); + bool release_lock = false; + + if (unlocked && mr->global_locking) { + qemu_mutex_lock_iothread(); + unlocked = false; + release_lock = true; + } + if (mr->flush_coalesced_mmio) { + if (unlocked) { + qemu_mutex_lock_iothread(); + } + qemu_flush_coalesced_mmio_buffer(); + if (unlocked) { + qemu_mutex_unlock_iothread(); + } + } + + return release_lock; +} + MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, uint8_t *buf, int len, bool is_write) { @@ -2325,6 +2361,7 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, hwaddr addr1; MemoryRegion *mr; MemTxResult result = MEMTX_OK; + bool release_lock = false; rcu_read_lock(); while (len > 0) { @@ -2333,6 +2370,7 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, if (is_write) { if (!memory_access_is_direct(mr, is_write)) { + release_lock |= prepare_mmio_access(mr); l = memory_access_size(mr, l, addr1); /* XXX: could force current_cpu to NULL to avoid potential bugs */ @@ -2374,6 +2412,7 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, } else { if (!memory_access_is_direct(mr, is_write)) { /* I/O case */ + release_lock |= prepare_mmio_access(mr); l = memory_access_size(mr, l, addr1); switch (l) { case 8: @@ -2409,6 +2448,12 @@ MemTxResult address_space_rw(AddressSpace *as, hwaddr addr, MemTxAttrs attrs, memcpy(buf, ptr, l); } } + + if (release_lock) { + qemu_mutex_unlock_iothread(); + release_lock = false; + } + len -= l; buf += l; addr += l; @@ -2458,7 +2503,7 @@ static inline void cpu_physical_memory_write_rom_internal(AddressSpace *as, if (!(memory_region_is_ram(mr) || memory_region_is_romd(mr))) { - /* do nothing */ + l = memory_access_size(mr, l, addr1); } else { addr1 += memory_region_get_ram_addr(mr); /* ROM/RAM case */ @@ -2735,10 +2780,13 @@ static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr, hwaddr l = 4; hwaddr addr1; MemTxResult r; + bool release_lock = false; rcu_read_lock(); mr = address_space_translate(as, addr, &addr1, &l, false); if (l < 4 || !memory_access_is_direct(mr, false)) { + release_lock |= prepare_mmio_access(mr); + /* I/O case */ r = memory_region_dispatch_read(mr, addr1, &val, 4, attrs); #if defined(TARGET_WORDS_BIGENDIAN) @@ -2771,6 +2819,9 @@ static inline uint32_t address_space_ldl_internal(AddressSpace *as, hwaddr addr, if (result) { *result = r; } + if (release_lock) { + qemu_mutex_unlock_iothread(); + } rcu_read_unlock(); return val; } @@ -2823,11 +2874,14 @@ static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr, hwaddr l = 8; hwaddr addr1; MemTxResult r; + bool release_lock = false; rcu_read_lock(); mr = address_space_translate(as, addr, &addr1, &l, false); if (l < 8 || !memory_access_is_direct(mr, false)) { + release_lock |= prepare_mmio_access(mr); + /* I/O case */ r = memory_region_dispatch_read(mr, addr1, &val, 8, attrs); #if defined(TARGET_WORDS_BIGENDIAN) @@ -2860,6 +2914,9 @@ static inline uint64_t address_space_ldq_internal(AddressSpace *as, hwaddr addr, if (result) { *result = r; } + if (release_lock) { + qemu_mutex_unlock_iothread(); + } rcu_read_unlock(); return val; } @@ -2932,11 +2989,14 @@ static inline uint32_t address_space_lduw_internal(AddressSpace *as, hwaddr l = 2; hwaddr addr1; MemTxResult r; + bool release_lock = false; rcu_read_lock(); mr = address_space_translate(as, addr, &addr1, &l, false); if (l < 2 || !memory_access_is_direct(mr, false)) { + release_lock |= prepare_mmio_access(mr); + /* I/O case */ r = memory_region_dispatch_read(mr, addr1, &val, 2, attrs); #if defined(TARGET_WORDS_BIGENDIAN) @@ -2969,6 +3029,9 @@ static inline uint32_t address_space_lduw_internal(AddressSpace *as, if (result) { *result = r; } + if (release_lock) { + qemu_mutex_unlock_iothread(); + } rcu_read_unlock(); return val; } @@ -3021,11 +3084,14 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, hwaddr addr1; MemTxResult r; uint8_t dirty_log_mask; + bool release_lock = false; rcu_read_lock(); mr = address_space_translate(as, addr, &addr1, &l, true); if (l < 4 || !memory_access_is_direct(mr, true)) { + release_lock |= prepare_mmio_access(mr); + r = memory_region_dispatch_write(mr, addr1, val, 4, attrs); } else { addr1 += memory_region_get_ram_addr(mr) & TARGET_PAGE_MASK; @@ -3040,6 +3106,9 @@ void address_space_stl_notdirty(AddressSpace *as, hwaddr addr, uint32_t val, if (result) { *result = r; } + if (release_lock) { + qemu_mutex_unlock_iothread(); + } rcu_read_unlock(); } @@ -3060,11 +3129,14 @@ static inline void address_space_stl_internal(AddressSpace *as, hwaddr l = 4; hwaddr addr1; MemTxResult r; + bool release_lock = false; rcu_read_lock(); mr = address_space_translate(as, addr, &addr1, &l, true); if (l < 4 || !memory_access_is_direct(mr, true)) { + release_lock |= prepare_mmio_access(mr); + #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap32(val); @@ -3096,6 +3168,9 @@ static inline void address_space_stl_internal(AddressSpace *as, if (result) { *result = r; } + if (release_lock) { + qemu_mutex_unlock_iothread(); + } rcu_read_unlock(); } @@ -3165,10 +3240,13 @@ static inline void address_space_stw_internal(AddressSpace *as, hwaddr l = 2; hwaddr addr1; MemTxResult r; + bool release_lock = false; rcu_read_lock(); mr = address_space_translate(as, addr, &addr1, &l, true); if (l < 2 || !memory_access_is_direct(mr, true)) { + release_lock |= prepare_mmio_access(mr); + #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap16(val); @@ -3200,6 +3278,9 @@ static inline void address_space_stw_internal(AddressSpace *as, if (result) { *result = r; } + if (release_lock) { + qemu_mutex_unlock_iothread(); + } rcu_read_unlock(); } diff --git a/hw/acpi/core.c b/hw/acpi/core.c index 0f201d8c6d..fe6215af4a 100644 --- a/hw/acpi/core.c +++ b/hw/acpi/core.c @@ -528,6 +528,7 @@ void acpi_pm_tmr_init(ACPIREGS *ar, acpi_update_sci_fn update_sci, ar->tmr.timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, acpi_pm_tmr_timer, ar); memory_region_init_io(&ar->tmr.io, memory_region_owner(parent), &acpi_pm_tmr_ops, ar, "acpi-tmr", 4); + memory_region_clear_global_locking(&ar->tmr.io); memory_region_add_subregion(parent, 8, &ar->tmr.io); } diff --git a/include/exec/cpu-all.h b/include/exec/cpu-all.h index ac06c6721c..8999634981 100644 --- a/include/exec/cpu-all.h +++ b/include/exec/cpu-all.h @@ -26,6 +26,12 @@ #include "qom/cpu.h" #include "qemu/rcu.h" +#define EXCP_INTERRUPT 0x10000 /* async interruption */ +#define EXCP_HLT 0x10001 /* hlt instruction reached */ +#define EXCP_DEBUG 0x10002 /* cpu stopped after a breakpoint or singlestep */ +#define EXCP_HALTED 0x10003 /* cpu is halted (waiting for external event) */ +#define EXCP_YIELD 0x10004 /* cpu wants to yield timeslice to another */ + /* some important defines: * * WORDS_ALIGNED : if defined, the host cpu can only make word aligned diff --git a/include/exec/cpu-common.h b/include/exec/cpu-common.h index de8a7200a9..9fb1d541d4 100644 --- a/include/exec/cpu-common.h +++ b/include/exec/cpu-common.h @@ -13,6 +13,8 @@ #include "qemu/bswap.h" #include "qemu/queue.h" +#include "qemu/fprintf-fn.h" +#include "qemu/typedefs.h" /** * CPUListState: diff --git a/include/exec/cpu-defs.h b/include/exec/cpu-defs.h index d5aecaf49e..98b9cff310 100644 --- a/include/exec/cpu-defs.h +++ b/include/exec/cpu-defs.h @@ -56,20 +56,6 @@ typedef uint64_t target_ulong; #error TARGET_LONG_SIZE undefined #endif -#define EXCP_INTERRUPT 0x10000 /* async interruption */ -#define EXCP_HLT 0x10001 /* hlt instruction reached */ -#define EXCP_DEBUG 0x10002 /* cpu stopped after a breakpoint or singlestep */ -#define EXCP_HALTED 0x10003 /* cpu is halted (waiting for external event) */ -#define EXCP_YIELD 0x10004 /* cpu wants to yield timeslice to another */ - -/* Only the bottom TB_JMP_PAGE_BITS of the jump cache hash bits vary for - addresses on the same page. The top bits are the same. This allows - TLB invalidation to quickly clear a subset of the hash table. */ -#define TB_JMP_PAGE_BITS (TB_JMP_CACHE_BITS / 2) -#define TB_JMP_PAGE_SIZE (1 << TB_JMP_PAGE_BITS) -#define TB_JMP_ADDR_MASK (TB_JMP_PAGE_SIZE - 1) -#define TB_JMP_PAGE_MASK (TB_JMP_CACHE_SIZE - TB_JMP_PAGE_SIZE) - #if !defined(CONFIG_USER_ONLY) /* use a fully associative victim tlb of 8 entries */ #define CPU_VTLB_SIZE 8 @@ -161,7 +147,6 @@ typedef struct CPUIOTLBEntry { #endif -#define CPU_TEMP_BUF_NLONGS 128 #define CPU_COMMON \ /* soft mmu support */ \ CPU_COMMON_TLB \ diff --git a/include/exec/exec-all.h b/include/exec/exec-all.h index 2573e8c36e..d678114cb2 100644 --- a/include/exec/exec-all.h +++ b/include/exec/exec-all.h @@ -195,26 +195,6 @@ struct TBContext { int tb_invalidated_flag; }; -static inline unsigned int tb_jmp_cache_hash_page(target_ulong pc) -{ - target_ulong tmp; - tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)); - return (tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK; -} - -static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc) -{ - target_ulong tmp; - tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)); - return (((tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK) - | (tmp & TB_JMP_ADDR_MASK)); -} - -static inline unsigned int tb_phys_hash_func(tb_page_addr_t pc) -{ - return (pc >> 2) & (CODE_GEN_PHYS_HASH_SIZE - 1); -} - void tb_free(TranslationBlock *tb); void tb_flush(CPUArchState *env); void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr); diff --git a/include/exec/memory.h b/include/exec/memory.h index 8ae004eb06..139471500f 100644 --- a/include/exec/memory.h +++ b/include/exec/memory.h @@ -23,7 +23,6 @@ #include <stdint.h> #include <stdbool.h> -#include "qemu-common.h" #include "exec/cpu-common.h" #ifndef CONFIG_USER_ONLY #include "exec/hwaddr.h" @@ -180,6 +179,7 @@ struct MemoryRegion { bool rom_device; bool warning_printed; /* For reservations */ bool flush_coalesced_mmio; + bool global_locking; MemoryRegion *alias; hwaddr alias_offset; int32_t priority; @@ -825,6 +825,31 @@ void memory_region_set_flush_coalesced(MemoryRegion *mr); void memory_region_clear_flush_coalesced(MemoryRegion *mr); /** + * memory_region_set_global_locking: Declares the access processing requires + * QEMU's global lock. + * + * When this is invoked, accesses to the memory region will be processed while + * holding the global lock of QEMU. This is the default behavior of memory + * regions. + * + * @mr: the memory region to be updated. + */ +void memory_region_set_global_locking(MemoryRegion *mr); + +/** + * memory_region_clear_global_locking: Declares that access processing does + * not depend on the QEMU global lock. + * + * By clearing this property, accesses to the memory region will be processed + * outside of QEMU's global lock (unless the lock is held on when issuing the + * access request). In this case, the device model implementing the access + * handlers is responsible for synchronization of concurrency. + * + * @mr: the memory region to be updated. + */ +void memory_region_clear_global_locking(MemoryRegion *mr); + +/** * memory_region_add_eventfd: Request an eventfd to be triggered when a word * is written to a location. * diff --git a/include/exec/tb-hash.h b/include/exec/tb-hash.h new file mode 100644 index 0000000000..0f4e8a08af --- /dev/null +++ b/include/exec/tb-hash.h @@ -0,0 +1,51 @@ +/* + * internal execution defines for qemu + * + * Copyright (c) 2003 Fabrice Bellard + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef EXEC_TB_HASH +#define EXEC_TB_HASH + +/* Only the bottom TB_JMP_PAGE_BITS of the jump cache hash bits vary for + addresses on the same page. The top bits are the same. This allows + TLB invalidation to quickly clear a subset of the hash table. */ +#define TB_JMP_PAGE_BITS (TB_JMP_CACHE_BITS / 2) +#define TB_JMP_PAGE_SIZE (1 << TB_JMP_PAGE_BITS) +#define TB_JMP_ADDR_MASK (TB_JMP_PAGE_SIZE - 1) +#define TB_JMP_PAGE_MASK (TB_JMP_CACHE_SIZE - TB_JMP_PAGE_SIZE) + +static inline unsigned int tb_jmp_cache_hash_page(target_ulong pc) +{ + target_ulong tmp; + tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)); + return (tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK; +} + +static inline unsigned int tb_jmp_cache_hash_func(target_ulong pc) +{ + target_ulong tmp; + tmp = pc ^ (pc >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)); + return (((tmp >> (TARGET_PAGE_BITS - TB_JMP_PAGE_BITS)) & TB_JMP_PAGE_MASK) + | (tmp & TB_JMP_ADDR_MASK)); +} + +static inline unsigned int tb_phys_hash_func(tb_page_addr_t pc) +{ + return (pc >> 2) & (CODE_GEN_PHYS_HASH_SIZE - 1); +} + +#endif diff --git a/include/hw/arm/arm.h b/include/hw/arm/arm.h index 760804cc46..4dcd4f9b63 100644 --- a/include/hw/arm/arm.h +++ b/include/hw/arm/arm.h @@ -14,6 +14,7 @@ #include "exec/memory.h" #include "hw/irq.h" #include "qemu/notify.h" +#include "cpu.h" /* armv7m.c */ qemu_irq *armv7m_init(MemoryRegion *system_memory, int mem_size, int num_irq, diff --git a/include/qemu-common.h b/include/qemu-common.h index d52d09cfb8..237d6547b3 100644 --- a/include/qemu-common.h +++ b/include/qemu-common.h @@ -15,6 +15,7 @@ #include "qemu/compiler.h" #include "config-host.h" #include "qemu/typedefs.h" +#include "qemu/fprintf-fn.h" #if defined(__arm__) || defined(__sparc__) || defined(__mips__) || defined(__hppa__) || defined(__ia64__) #define WORDS_ALIGNED @@ -85,9 +86,6 @@ # error Unknown pointer size #endif -typedef int (*fprintf_function)(FILE *f, const char *fmt, ...) - GCC_FMT_ATTR(2, 3); - #ifdef _WIN32 #define fsync _commit #if !defined(lseek) @@ -455,6 +453,7 @@ void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size); #define VECTYPE __vector unsigned char #define SPLAT(p) vec_splat(vec_ld(0, p), 0) #define ALL_EQ(v1, v2) vec_all_eq(v1, v2) +#define VEC_OR(v1, v2) ((v1) | (v2)) /* altivec.h may redefine the bool macro as vector type. * Reset it to POSIX semantics. */ #define bool _Bool @@ -463,10 +462,12 @@ void qemu_hexdump(const char *buf, FILE *fp, const char *prefix, size_t size); #define VECTYPE __m128i #define SPLAT(p) _mm_set1_epi8(*(p)) #define ALL_EQ(v1, v2) (_mm_movemask_epi8(_mm_cmpeq_epi8(v1, v2)) == 0xFFFF) +#define VEC_OR(v1, v2) (_mm_or_si128(v1, v2)) #else #define VECTYPE unsigned long #define SPLAT(p) (*(p) * (~0UL / 255)) #define ALL_EQ(v1, v2) ((v1) == (v2)) +#define VEC_OR(v1, v2) ((v1) | (v2)) #endif #define BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR 8 diff --git a/include/qemu/fprintf-fn.h b/include/qemu/fprintf-fn.h new file mode 100644 index 0000000000..9ddc90f1c5 --- /dev/null +++ b/include/qemu/fprintf-fn.h @@ -0,0 +1,17 @@ +/* + * Typedef for fprintf-alike function pointers. + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef QEMU_FPRINTF_FN_H +#define QEMU_FPRINTF_FN_H 1 + +#include "qemu/compiler.h" +#include <stdio.h> + +typedef int (*fprintf_function)(FILE *f, const char *fmt, ...) + GCC_FMT_ATTR(2, 3); + +#endif diff --git a/include/qemu/main-loop.h b/include/qemu/main-loop.h index 0f4a0fd4b2..bc18ca30e4 100644 --- a/include/qemu/main-loop.h +++ b/include/qemu/main-loop.h @@ -223,6 +223,16 @@ int qemu_add_child_watch(pid_t pid); #endif /** + * qemu_mutex_iothread_locked: Return lock status of the main loop mutex. + * + * The main loop mutex is the coarsest lock in QEMU, and as such it + * must always be taken outside other locks. This function helps + * functions take different paths depending on whether the current + * thread is running within the main loop mutex. + */ +bool qemu_mutex_iothread_locked(void); + +/** * qemu_mutex_lock_iothread: Lock the main loop mutex. * * This function locks the main loop mutex. The mutex is taken by @@ -1099,9 +1099,17 @@ static int kvm_irqchip_get_virq(KVMState *s) uint32_t *word = s->used_gsi_bitmap; int max_words = ALIGN(s->gsi_count, 32) / 32; int i, zeroes; - bool retry = true; -again: + /* + * PIC and IOAPIC share the first 16 GSI numbers, thus the available + * GSI numbers are more than the number of IRQ route. Allocating a GSI + * number can succeed even though a new route entry cannot be added. + * When this happens, flush dynamic MSI entries to free IRQ route entries. + */ + if (!s->direct_msi && s->irq_routes->nr == s->gsi_count) { + kvm_flush_dynamic_msi_routes(s); + } + /* Return the lowest unused GSI in the bitmap */ for (i = 0; i < max_words; i++) { zeroes = ctz32(~word[i]); @@ -1111,11 +1119,6 @@ again: return zeroes + i * 32; } - if (!s->direct_msi && retry) { - retry = false; - kvm_flush_dynamic_msi_routes(s); - goto again; - } return -ENOSPC; } @@ -1752,6 +1755,8 @@ int kvm_cpu_exec(CPUState *cpu) return EXCP_HLT; } + qemu_mutex_unlock_iothread(); + do { MemTxAttrs attrs; @@ -1770,11 +1775,9 @@ int kvm_cpu_exec(CPUState *cpu) */ qemu_cpu_kick_self(); } - qemu_mutex_unlock_iothread(); run_ret = kvm_vcpu_ioctl(cpu, KVM_RUN, 0); - qemu_mutex_lock_iothread(); attrs = kvm_arch_post_run(cpu, run); if (run_ret < 0) { @@ -1801,6 +1804,7 @@ int kvm_cpu_exec(CPUState *cpu) switch (run->exit_reason) { case KVM_EXIT_IO: DPRINTF("handle_io\n"); + /* Called outside BQL */ kvm_handle_io(run->io.port, attrs, (uint8_t *)run + run->io.data_offset, run->io.direction, @@ -1810,6 +1814,7 @@ int kvm_cpu_exec(CPUState *cpu) break; case KVM_EXIT_MMIO: DPRINTF("handle_mmio\n"); + /* Called outside BQL */ address_space_rw(&address_space_memory, run->mmio.phys_addr, attrs, run->mmio.data, @@ -1857,6 +1862,8 @@ int kvm_cpu_exec(CPUState *cpu) } } while (ret == 0); + qemu_mutex_lock_iothread(); + if (ret < 0) { cpu_dump_state(cpu, stderr, fprintf, CPU_DUMP_CODE); vm_stop(RUN_STATE_INTERNAL_ERROR); @@ -396,9 +396,6 @@ static MemTxResult memory_region_read_accessor(MemoryRegion *mr, { uint64_t tmp; - if (mr->flush_coalesced_mmio) { - qemu_flush_coalesced_mmio_buffer(); - } tmp = mr->ops->read(mr->opaque, addr, size); trace_memory_region_ops_read(mr, addr, tmp, size); *value |= (tmp & mask) << shift; @@ -416,9 +413,6 @@ static MemTxResult memory_region_read_with_attrs_accessor(MemoryRegion *mr, uint64_t tmp = 0; MemTxResult r; - if (mr->flush_coalesced_mmio) { - qemu_flush_coalesced_mmio_buffer(); - } r = mr->ops->read_with_attrs(mr->opaque, addr, &tmp, size, attrs); trace_memory_region_ops_read(mr, addr, tmp, size); *value |= (tmp & mask) << shift; @@ -451,9 +445,6 @@ static MemTxResult memory_region_write_accessor(MemoryRegion *mr, { uint64_t tmp; - if (mr->flush_coalesced_mmio) { - qemu_flush_coalesced_mmio_buffer(); - } tmp = (*value >> shift) & mask; trace_memory_region_ops_write(mr, addr, tmp, size); mr->ops->write(mr->opaque, addr, tmp, size); @@ -470,9 +461,6 @@ static MemTxResult memory_region_write_with_attrs_accessor(MemoryRegion *mr, { uint64_t tmp; - if (mr->flush_coalesced_mmio) { - qemu_flush_coalesced_mmio_buffer(); - } tmp = (*value >> shift) & mask; trace_memory_region_ops_write(mr, addr, tmp, size); return mr->ops->write_with_attrs(mr->opaque, addr, tmp, size, attrs); @@ -1012,6 +1000,7 @@ static void memory_region_initfn(Object *obj) mr->ram_addr = RAM_ADDR_INVALID; mr->enabled = true; mr->romd_mode = true; + mr->global_locking = true; mr->destructor = memory_region_destructor_none; QTAILQ_INIT(&mr->subregions); QTAILQ_INIT(&mr->coalesced); @@ -1646,6 +1635,16 @@ void memory_region_clear_flush_coalesced(MemoryRegion *mr) } } +void memory_region_set_global_locking(MemoryRegion *mr) +{ + mr->global_locking = true; +} + +void memory_region_clear_global_locking(MemoryRegion *mr) +{ + mr->global_locking = false; +} + void memory_region_add_eventfd(MemoryRegion *mr, hwaddr addr, unsigned size, diff --git a/memory_mapping.c b/memory_mapping.c index 7b69801cb8..36d6b26046 100644 --- a/memory_mapping.c +++ b/memory_mapping.c @@ -13,8 +13,8 @@ #include <glib.h> +#include "qemu-common.h" #include "cpu.h" -#include "exec/cpu-all.h" #include "sysemu/memory_mapping.h" #include "exec/memory.h" #include "exec/address-spaces.h" diff --git a/stubs/iothread-lock.c b/stubs/iothread-lock.c index 5d8aca1b37..dda6f6b58d 100644 --- a/stubs/iothread-lock.c +++ b/stubs/iothread-lock.c @@ -1,6 +1,11 @@ #include "qemu-common.h" #include "qemu/main-loop.h" +bool qemu_mutex_iothread_locked(void) +{ + return true; +} + void qemu_mutex_lock_iothread(void) { } diff --git a/target-i386/kvm.c b/target-i386/kvm.c index daced5cb94..6426600c63 100644 --- a/target-i386/kvm.c +++ b/target-i386/kvm.c @@ -2191,7 +2191,10 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) /* Inject NMI */ if (cpu->interrupt_request & CPU_INTERRUPT_NMI) { + qemu_mutex_lock_iothread(); cpu->interrupt_request &= ~CPU_INTERRUPT_NMI; + qemu_mutex_unlock_iothread(); + DPRINTF("injected NMI\n"); ret = kvm_vcpu_ioctl(cpu, KVM_NMI); if (ret < 0) { @@ -2200,6 +2203,10 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) } } + if (!kvm_irqchip_in_kernel()) { + qemu_mutex_lock_iothread(); + } + /* Force the VCPU out of its inner loop to process any INIT requests * or (for userspace APIC, but it is cheap to combine the checks here) * pending TPR access reports. @@ -2243,6 +2250,8 @@ void kvm_arch_pre_run(CPUState *cpu, struct kvm_run *run) DPRINTF("setting tpr\n"); run->cr8 = cpu_get_apic_tpr(x86_cpu->apic_state); + + qemu_mutex_unlock_iothread(); } } @@ -2256,8 +2265,17 @@ MemTxAttrs kvm_arch_post_run(CPUState *cpu, struct kvm_run *run) } else { env->eflags &= ~IF_MASK; } + + /* We need to protect the apic state against concurrent accesses from + * different threads in case the userspace irqchip is used. */ + if (!kvm_irqchip_in_kernel()) { + qemu_mutex_lock_iothread(); + } cpu_set_apic_tpr(x86_cpu->apic_state, run->cr8); cpu_set_apic_base(x86_cpu->apic_state, run->apic_base); + if (!kvm_irqchip_in_kernel()) { + qemu_mutex_unlock_iothread(); + } return cpu_get_mem_attrs(env); } @@ -2550,13 +2568,17 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) switch (run->exit_reason) { case KVM_EXIT_HLT: DPRINTF("handle_hlt\n"); + qemu_mutex_lock_iothread(); ret = kvm_handle_halt(cpu); + qemu_mutex_unlock_iothread(); break; case KVM_EXIT_SET_TPR: ret = 0; break; case KVM_EXIT_TPR_ACCESS: + qemu_mutex_lock_iothread(); ret = kvm_handle_tpr_access(cpu); + qemu_mutex_unlock_iothread(); break; case KVM_EXIT_FAIL_ENTRY: code = run->fail_entry.hardware_entry_failure_reason; @@ -2582,7 +2604,9 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) break; case KVM_EXIT_DEBUG: DPRINTF("kvm_exit_debug\n"); + qemu_mutex_lock_iothread(); ret = kvm_handle_debug(cpu, &run->debug.arch); + qemu_mutex_unlock_iothread(); break; default: fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason); diff --git a/target-mips/kvm.c b/target-mips/kvm.c index 948619fbab..7d2293d934 100644 --- a/target-mips/kvm.c +++ b/target-mips/kvm.c @@ -99,6 +99,8 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run) int r; struct kvm_mips_interrupt intr; + qemu_mutex_lock_iothread(); + if ((cs->interrupt_request & CPU_INTERRUPT_HARD) && cpu_mips_io_interrupts_pending(cpu)) { intr.cpu = -1; @@ -109,6 +111,8 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run) __func__, cs->cpu_index, intr.irq); } } + + qemu_mutex_unlock_iothread(); } MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run) diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c index afb4696b8a..ddf469fe09 100644 --- a/target-ppc/kvm.c +++ b/target-ppc/kvm.c @@ -1242,6 +1242,8 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run) int r; unsigned irq; + qemu_mutex_lock_iothread(); + /* PowerPC QEMU tracks the various core input pins (interrupt, critical * interrupt, reset, etc) in PPC-specific env->irq_input_state. */ if (!cap_interrupt_level && @@ -1269,6 +1271,8 @@ void kvm_arch_pre_run(CPUState *cs, struct kvm_run *run) /* We don't know if there are more interrupts pending after this. However, * the guest will return to userspace in the course of handling this one * anyways, so we will get a chance to deliver the rest. */ + + qemu_mutex_unlock_iothread(); } MemTxAttrs kvm_arch_post_run(CPUState *cs, struct kvm_run *run) @@ -1570,6 +1574,8 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) CPUPPCState *env = &cpu->env; int ret; + qemu_mutex_lock_iothread(); + switch (run->exit_reason) { case KVM_EXIT_DCR: if (run->dcr.is_write) { @@ -1620,6 +1626,7 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) break; } + qemu_mutex_unlock_iothread(); return ret; } diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c index 135111a2c4..ae3a0affec 100644 --- a/target-s390x/kvm.c +++ b/target-s390x/kvm.c @@ -2007,6 +2007,8 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) S390CPU *cpu = S390_CPU(cs); int ret = 0; + qemu_mutex_lock_iothread(); + switch (run->exit_reason) { case KVM_EXIT_S390_SIEIC: ret = handle_intercept(cpu); @@ -2027,6 +2029,7 @@ int kvm_arch_handle_exit(CPUState *cs, struct kvm_run *run) fprintf(stderr, "Unknown KVM exit: %d\n", run->exit_reason); break; } + qemu_mutex_unlock_iothread(); if (ret == 0) { ret = EXCP_INTERRUPT; diff --git a/target-s390x/mmu_helper.c b/target-s390x/mmu_helper.c index 815ff42dde..1ea6d812c2 100644 --- a/target-s390x/mmu_helper.c +++ b/target-s390x/mmu_helper.c @@ -17,8 +17,8 @@ #include "qemu/error-report.h" #include "exec/address-spaces.h" -#include "sysemu/kvm.h" #include "cpu.h" +#include "sysemu/kvm.h" /* #define DEBUG_S390 */ /* #define DEBUG_S390_PTE */ @@ -29,6 +29,8 @@ #include "qemu/bitops.h" #include "tcg-target.h" +#define CPU_TEMP_BUF_NLONGS 128 + /* Default target word size to pointer size. */ #ifndef TCG_TARGET_REG_BITS # if UINTPTR_MAX == UINT32_MAX diff --git a/translate-all.c b/translate-all.c index b6b0e1c098..412bc9005f 100644 --- a/translate-all.c +++ b/translate-all.c @@ -58,6 +58,7 @@ #endif #include "exec/cputlb.h" +#include "exec/tb-hash.h" #include "translate-all.h" #include "qemu/bitmap.h" #include "qemu/timer.h" diff --git a/util/cutils.c b/util/cutils.c index 144b25c05a..5d1c9ebe05 100644 --- a/util/cutils.c +++ b/util/cutils.c @@ -207,13 +207,13 @@ size_t buffer_find_nonzero_offset(const void *buf, size_t len) for (i = BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR; i < len / sizeof(VECTYPE); i += BUFFER_FIND_NONZERO_OFFSET_UNROLL_FACTOR) { - VECTYPE tmp0 = p[i + 0] | p[i + 1]; - VECTYPE tmp1 = p[i + 2] | p[i + 3]; - VECTYPE tmp2 = p[i + 4] | p[i + 5]; - VECTYPE tmp3 = p[i + 6] | p[i + 7]; - VECTYPE tmp01 = tmp0 | tmp1; - VECTYPE tmp23 = tmp2 | tmp3; - if (!ALL_EQ(tmp01 | tmp23, zero)) { + VECTYPE tmp0 = VEC_OR(p[i + 0], p[i + 1]); + VECTYPE tmp1 = VEC_OR(p[i + 2], p[i + 3]); + VECTYPE tmp2 = VEC_OR(p[i + 4], p[i + 5]); + VECTYPE tmp3 = VEC_OR(p[i + 6], p[i + 7]); + VECTYPE tmp01 = VEC_OR(tmp0, tmp1); + VECTYPE tmp23 = VEC_OR(tmp2, tmp3); + if (!ALL_EQ(VEC_OR(tmp01, tmp23), zero)) { break; } } |