diff options
40 files changed, 3066 insertions, 562 deletions
diff --git a/.gitignore b/.gitignore index c72955aad1..81b1510297 100644 --- a/.gitignore +++ b/.gitignore @@ -77,6 +77,7 @@ pc-bios/vgabios-pq/status pc-bios/optionrom/linuxboot.bin pc-bios/optionrom/multiboot.bin pc-bios/optionrom/multiboot.raw +pc-bios/optionrom/kvmvapic.bin .stgit-* cscope.* tags @@ -256,7 +256,7 @@ pxe-e1000.rom pxe-eepro100.rom pxe-ne2k_pci.rom \ pxe-pcnet.rom pxe-rtl8139.rom pxe-virtio.rom \ bamboo.dtb petalogix-s3adsp1800.dtb petalogix-ml605.dtb \ mpc8544ds.dtb \ -multiboot.bin linuxboot.bin \ +multiboot.bin linuxboot.bin kvmvapic.bin \ s390-zipl.rom \ spapr-rtas.bin slof.bin \ palcode-clipper diff --git a/Makefile.target b/Makefile.target index 6a56b3e4b9..343eb78494 100644 --- a/Makefile.target +++ b/Makefile.target @@ -237,7 +237,8 @@ obj-y += device-hotplug.o # Hardware support obj-i386-y += mc146818rtc.o pc.o -obj-i386-y += sga.o apic_common.o apic.o ioapic_common.o ioapic.o piix_pci.o +obj-i386-y += apic_common.o apic.o kvmvapic.o +obj-i386-y += sga.o ioapic_common.o ioapic.o piix_pci.o obj-i386-y += vmport.o obj-i386-y += pci-hotplug.o smbios.o wdt_ib700.o obj-i386-y += debugcon.o multiboot.o @@ -375,8 +375,9 @@ DECLARE_TLS(CPUState *,cpu_single_env); #define CPU_INTERRUPT_TGT_INT_0 0x0100 #define CPU_INTERRUPT_TGT_INT_1 0x0400 #define CPU_INTERRUPT_TGT_INT_2 0x0800 +#define CPU_INTERRUPT_TGT_INT_3 0x2000 -/* First unused bit: 0x2000. */ +/* First unused bit: 0x4000. */ /* The set of all bits that should be masked when single-stepping. */ #define CPU_INTERRUPT_SSTEP_MASK \ @@ -714,6 +714,7 @@ static void *qemu_kvm_cpu_thread_fn(void *arg) qemu_mutex_lock(&qemu_global_mutex); qemu_thread_get_self(env->thread); env->thread_id = qemu_get_thread_id(); + cpu_single_env = env; r = kvm_init_vcpu(env); if (r < 0) { @@ -760,6 +761,11 @@ static void *qemu_tcg_cpu_thread_fn(void *arg) /* wait for initial kick-off after machine start */ while (first_cpu->stopped) { qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex); + + /* process any pending work */ + for (env = first_cpu; env != NULL; env = env->next_cpu) { + qemu_wait_io_event_common(env); + } } while (1) { @@ -852,7 +858,7 @@ static int all_vcpus_paused(void) if (!penv->stopped) { return 0; } - penv = (CPUState *)penv->next_cpu; + penv = penv->next_cpu; } return 1; @@ -866,7 +872,19 @@ void pause_all_vcpus(void) while (penv) { penv->stop = 1; qemu_cpu_kick(penv); - penv = (CPUState *)penv->next_cpu; + penv = penv->next_cpu; + } + + if (!qemu_thread_is_self(&io_thread)) { + cpu_stop_current(); + if (!kvm_enabled()) { + while (penv) { + penv->stop = 0; + penv->stopped = 1; + penv = penv->next_cpu; + } + return; + } } while (!all_vcpus_paused()) { @@ -874,7 +892,7 @@ void pause_all_vcpus(void) penv = first_cpu; while (penv) { qemu_cpu_kick(penv); - penv = (CPUState *)penv->next_cpu; + penv = penv->next_cpu; } } } @@ -888,7 +906,7 @@ void resume_all_vcpus(void) penv->stop = 0; penv->stopped = 0; qemu_cpu_kick(penv); - penv = (CPUState *)penv->next_cpu; + penv = penv->next_cpu; } } diff --git a/exec-obsolete.h b/exec-obsolete.h index 94c23d0951..4dbe4768aa 100644 --- a/exec-obsolete.h +++ b/exec-obsolete.h @@ -37,7 +37,7 @@ void cpu_unregister_io_memory(int table_address); struct MemoryRegionSection; void cpu_register_physical_memory_log(struct MemoryRegionSection *section, - bool readable, bool readonly); + bool readonly); void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size); void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size); @@ -121,6 +121,9 @@ static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start, void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end, int dirty_flags); + +extern const IORangeOps memory_region_iorange_ops; + #endif #endif @@ -160,29 +160,21 @@ typedef struct PageDesc { #define L2_BITS 10 #define L2_SIZE (1 << L2_BITS) +#define P_L2_LEVELS \ + (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1) + /* The bits remaining after N lower levels of page tables. */ -#define P_L1_BITS_REM \ - ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS) #define V_L1_BITS_REM \ ((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS) -/* Size of the L1 page table. Avoid silly small sizes. */ -#if P_L1_BITS_REM < 4 -#define P_L1_BITS (P_L1_BITS_REM + L2_BITS) -#else -#define P_L1_BITS P_L1_BITS_REM -#endif - #if V_L1_BITS_REM < 4 #define V_L1_BITS (V_L1_BITS_REM + L2_BITS) #else #define V_L1_BITS V_L1_BITS_REM #endif -#define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS) #define V_L1_SIZE ((target_ulong)1 << V_L1_BITS) -#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS) #define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS) unsigned long qemu_real_host_page_size; @@ -194,15 +186,27 @@ unsigned long qemu_host_page_mask; static void *l1_map[V_L1_SIZE]; #if !defined(CONFIG_USER_ONLY) -typedef struct PhysPageDesc { - /* offset in host memory of the page + io_index in the low bits */ - ram_addr_t phys_offset; - ram_addr_t region_offset; -} PhysPageDesc; +typedef struct PhysPageEntry PhysPageEntry; + +static MemoryRegionSection *phys_sections; +static unsigned phys_sections_nb, phys_sections_nb_alloc; +static uint16_t phys_section_unassigned; + +struct PhysPageEntry { + uint16_t is_leaf : 1; + /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */ + uint16_t ptr : 15; +}; + +/* Simple allocator for PhysPageEntry nodes */ +static PhysPageEntry (*phys_map_nodes)[L2_SIZE]; +static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc; + +#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1) /* This is a multi-level map on the physical address space. - The bottom level has pointers to PhysPageDesc. */ -static void *l1_phys_map[P_L1_SIZE]; + The bottom level has pointers to MemoryRegionSections. */ +static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 }; static void io_mem_init(void); static void memory_map_init(void); @@ -398,59 +402,112 @@ static inline PageDesc *page_find(tb_page_addr_t index) } #if !defined(CONFIG_USER_ONLY) -static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc) + +static void phys_map_node_reserve(unsigned nodes) { - PhysPageDesc *pd; - void **lp; - int i; + if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) { + typedef PhysPageEntry Node[L2_SIZE]; + phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16); + phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc, + phys_map_nodes_nb + nodes); + phys_map_nodes = g_renew(Node, phys_map_nodes, + phys_map_nodes_nb_alloc); + } +} - /* Level 1. Always allocated. */ - lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1)); +static uint16_t phys_map_node_alloc(void) +{ + unsigned i; + uint16_t ret; - /* Level 2..N-1. */ - for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) { - void **p = *lp; - if (p == NULL) { - if (!alloc) { - return NULL; - } - *lp = p = g_malloc0(sizeof(void *) * L2_SIZE); - } - lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1)); + ret = phys_map_nodes_nb++; + assert(ret != PHYS_MAP_NODE_NIL); + assert(ret != phys_map_nodes_nb_alloc); + for (i = 0; i < L2_SIZE; ++i) { + phys_map_nodes[ret][i].is_leaf = 0; + phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL; } + return ret; +} - pd = *lp; - if (pd == NULL) { - int i; - int first_index = index & ~(L2_SIZE - 1); +static void phys_map_nodes_reset(void) +{ + phys_map_nodes_nb = 0; +} - if (!alloc) { - return NULL; - } - *lp = pd = g_malloc(sizeof(PhysPageDesc) * L2_SIZE); +static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index, + target_phys_addr_t *nb, uint16_t leaf, + int level) +{ + PhysPageEntry *p; + int i; + target_phys_addr_t step = (target_phys_addr_t)1 << (level * L2_BITS); + + if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) { + lp->ptr = phys_map_node_alloc(); + p = phys_map_nodes[lp->ptr]; + if (level == 0) { + for (i = 0; i < L2_SIZE; i++) { + p[i].is_leaf = 1; + p[i].ptr = phys_section_unassigned; + } + } + } else { + p = phys_map_nodes[lp->ptr]; + } + lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)]; - for (i = 0; i < L2_SIZE; i++) { - pd[i].phys_offset = io_mem_unassigned.ram_addr; - pd[i].region_offset = (first_index + i) << TARGET_PAGE_BITS; + while (*nb && lp < &p[L2_SIZE]) { + if ((*index & (step - 1)) == 0 && *nb >= step) { + lp->is_leaf = true; + lp->ptr = leaf; + *index += step; + *nb -= step; + } else { + phys_page_set_level(lp, index, nb, leaf, level - 1); } + ++lp; } +} - return pd + (index & (L2_SIZE - 1)); +static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb, + uint16_t leaf) +{ + /* Wildly overreserve - it doesn't matter much. */ + phys_map_node_reserve(3 * P_L2_LEVELS); + + phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1); } -static inline PhysPageDesc phys_page_find(target_phys_addr_t index) +static MemoryRegionSection phys_page_find(target_phys_addr_t index) { - PhysPageDesc *p = phys_page_find_alloc(index, 0); + PhysPageEntry lp = phys_map; + PhysPageEntry *p; + int i; + MemoryRegionSection section; + target_phys_addr_t delta; + uint16_t s_index = phys_section_unassigned; - if (p) { - return *p; - } else { - return (PhysPageDesc) { - .phys_offset = io_mem_unassigned.ram_addr, - .region_offset = index << TARGET_PAGE_BITS, - }; + for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) { + if (lp.ptr == PHYS_MAP_NODE_NIL) { + goto not_found; + } + p = phys_map_nodes[lp.ptr]; + lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)]; } + + s_index = lp.ptr; +not_found: + section = phys_sections[s_index]; + index <<= TARGET_PAGE_BITS; + assert(section.offset_within_address_space <= index + && index <= section.offset_within_address_space + section.size-1); + delta = index - section.offset_within_address_space; + section.offset_within_address_space += delta; + section.offset_within_region += delta; + section.size -= delta; + return section; } static void tlb_protect_code(ram_addr_t ram_addr); @@ -1410,14 +1467,18 @@ static void breakpoint_invalidate(CPUState *env, target_ulong pc) static void breakpoint_invalidate(CPUState *env, target_ulong pc) { target_phys_addr_t addr; - target_ulong pd; ram_addr_t ram_addr; - PhysPageDesc p; + MemoryRegionSection section; addr = cpu_get_phys_page_debug(env, pc); - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; - ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK); + section = phys_page_find(addr >> TARGET_PAGE_BITS); + if (!(memory_region_is_ram(section.mr) + || (section.mr->rom_device && section.mr->readable))) { + return; + } + ram_addr = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) & TARGET_PAGE_MASK; + ram_addr |= (pc & ~TARGET_PAGE_MASK); tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0); } #endif @@ -1443,7 +1504,8 @@ int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len, CPUWatchpoint *wp; /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */ - if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) { + if ((len & (len - 1)) || (addr & ~len_mask) || + len == 0 || len > TARGET_PAGE_SIZE) { fprintf(stderr, "qemu: tried to set invalid watchpoint at " TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len); return -EINVAL; @@ -2095,24 +2157,21 @@ static void tlb_add_large_page(CPUState *env, target_ulong vaddr, env->tlb_flush_mask = mask; } -static bool is_ram_rom(ram_addr_t pd) +static bool is_ram_rom(MemoryRegionSection *s) { - pd &= ~TARGET_PAGE_MASK; - return pd == io_mem_ram.ram_addr || pd == io_mem_rom.ram_addr; + return memory_region_is_ram(s->mr); } -static bool is_romd(ram_addr_t pd) +static bool is_romd(MemoryRegionSection *s) { - MemoryRegion *mr; + MemoryRegion *mr = s->mr; - pd &= ~TARGET_PAGE_MASK; - mr = io_mem_region[pd]; return mr->rom_device && mr->readable; } -static bool is_ram_rom_romd(ram_addr_t pd) +static bool is_ram_rom_romd(MemoryRegionSection *s) { - return is_ram_rom(pd) || is_romd(pd); + return is_ram_rom(s) || is_romd(s); } /* Add a new TLB entry. At most one entry for a given virtual address @@ -2122,8 +2181,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, target_phys_addr_t paddr, int prot, int mmu_idx, target_ulong size) { - PhysPageDesc p; - unsigned long pd; + MemoryRegionSection section; unsigned int index; target_ulong address; target_ulong code_address; @@ -2136,8 +2194,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, if (size != TARGET_PAGE_SIZE) { tlb_add_large_page(env, vaddr, size); } - p = phys_page_find(paddr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(paddr >> TARGET_PAGE_BITS); #if defined(DEBUG_TLB) printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx " prot=%x idx=%d pd=0x%08lx\n", @@ -2145,15 +2202,21 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, #endif address = vaddr; - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* IO memory case (romd handled later) */ address |= TLB_MMIO; } - addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK); - if (is_ram_rom(pd)) { + if (is_ram_rom_romd(§ion)) { + addend = (unsigned long)(memory_region_get_ram_ptr(section.mr) + + section.offset_within_region); + } else { + addend = 0; + } + if (is_ram_rom(§ion)) { /* Normal RAM. */ - iotlb = pd & TARGET_PAGE_MASK; - if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr) + iotlb = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) & TARGET_PAGE_MASK; + if (!section.readonly) iotlb |= io_mem_notdirty.ram_addr; else iotlb |= io_mem_rom.ram_addr; @@ -2164,8 +2227,8 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, and avoid full address decoding in every device. We can't use the high bits of pd for this because IO_MEM_ROMD uses these as a ram address. */ - iotlb = (pd & ~TARGET_PAGE_MASK); - iotlb += p.region_offset; + iotlb = memory_region_get_ram_addr(section.mr) & ~TARGET_PAGE_MASK; + iotlb += section.offset_within_region; } code_address = address; @@ -2198,11 +2261,14 @@ void tlb_set_page(CPUState *env, target_ulong vaddr, te->addr_code = -1; } if (prot & PAGE_WRITE) { - if ((pd & ~TARGET_PAGE_MASK) == io_mem_rom.ram_addr || is_romd(pd)) { + if ((memory_region_is_ram(section.mr) && section.readonly) + || is_romd(§ion)) { /* Write access calls the I/O callback. */ te->addr_write = address | TLB_MMIO; - } else if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr && - !cpu_physical_memory_is_dirty(pd)) { + } else if (memory_region_is_ram(section.mr) + && !cpu_physical_memory_is_dirty( + section.mr->ram_addr + + section.offset_within_region)) { te->addr_write = address | TLB_NOTDIRTY; } else { te->addr_write = address; @@ -2491,34 +2557,66 @@ static inline void tlb_set_dirty(CPUState *env, typedef struct subpage_t { MemoryRegion iomem; target_phys_addr_t base; - ram_addr_t sub_io_index[TARGET_PAGE_SIZE]; - ram_addr_t region_offset[TARGET_PAGE_SIZE]; + uint16_t sub_section[TARGET_PAGE_SIZE]; } subpage_t; static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, - ram_addr_t memory, ram_addr_t region_offset); -static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, - ram_addr_t orig_memory, - ram_addr_t region_offset); -#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \ - need_subpage) \ - do { \ - if (addr > start_addr) \ - start_addr2 = 0; \ - else { \ - start_addr2 = start_addr & ~TARGET_PAGE_MASK; \ - if (start_addr2 > 0) \ - need_subpage = 1; \ - } \ - \ - if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \ - end_addr2 = TARGET_PAGE_SIZE - 1; \ - else { \ - end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \ - if (end_addr2 < TARGET_PAGE_SIZE - 1) \ - need_subpage = 1; \ - } \ - } while (0) + uint16_t section); +static subpage_t *subpage_init(target_phys_addr_t base); +static void destroy_page_desc(uint16_t section_index) +{ + MemoryRegionSection *section = &phys_sections[section_index]; + MemoryRegion *mr = section->mr; + + if (mr->subpage) { + subpage_t *subpage = container_of(mr, subpage_t, iomem); + memory_region_destroy(&subpage->iomem); + g_free(subpage); + } +} + +static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level) +{ + unsigned i; + PhysPageEntry *p; + + if (lp->ptr == PHYS_MAP_NODE_NIL) { + return; + } + + p = phys_map_nodes[lp->ptr]; + for (i = 0; i < L2_SIZE; ++i) { + if (!p[i].is_leaf) { + destroy_l2_mapping(&p[i], level - 1); + } else { + destroy_page_desc(p[i].ptr); + } + } + lp->is_leaf = 0; + lp->ptr = PHYS_MAP_NODE_NIL; +} + +static void destroy_all_mappings(void) +{ + destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1); + phys_map_nodes_reset(); +} + +static uint16_t phys_section_add(MemoryRegionSection *section) +{ + if (phys_sections_nb == phys_sections_nb_alloc) { + phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16); + phys_sections = g_renew(MemoryRegionSection, phys_sections, + phys_sections_nb_alloc); + } + phys_sections[phys_sections_nb] = *section; + return phys_sections_nb++; +} + +static void phys_sections_clear(void) +{ + phys_sections_nb = 0; +} /* register physical memory. For RAM, 'size' must be a multiple of the target page size. @@ -2528,101 +2626,78 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, start_addr and region_offset are rounded down to a page boundary before calculating this offset. This should not be a problem unless the low bits of start_addr and region_offset differ. */ -void cpu_register_physical_memory_log(MemoryRegionSection *section, - bool readable, bool readonly) +static void register_subpage(MemoryRegionSection *section) { - target_phys_addr_t start_addr = section->offset_within_address_space; - ram_addr_t size = section->size; - ram_addr_t phys_offset = section->mr->ram_addr; - ram_addr_t region_offset = section->offset_within_region; - target_phys_addr_t addr, end_addr; - PhysPageDesc *p; - CPUState *env; - ram_addr_t orig_size = size; subpage_t *subpage; - - if (memory_region_is_ram(section->mr)) { - phys_offset += region_offset; - region_offset = 0; + target_phys_addr_t base = section->offset_within_address_space + & TARGET_PAGE_MASK; + MemoryRegionSection existing = phys_page_find(base >> TARGET_PAGE_BITS); + MemoryRegionSection subsection = { + .offset_within_address_space = base, + .size = TARGET_PAGE_SIZE, + }; + target_phys_addr_t start, end; + + assert(existing.mr->subpage || existing.mr == &io_mem_unassigned); + + if (!(existing.mr->subpage)) { + subpage = subpage_init(base); + subsection.mr = &subpage->iomem; + phys_page_set(base >> TARGET_PAGE_BITS, 1, + phys_section_add(&subsection)); + } else { + subpage = container_of(existing.mr, subpage_t, iomem); } + start = section->offset_within_address_space & ~TARGET_PAGE_MASK; + end = start + section->size; + subpage_register(subpage, start, end, phys_section_add(section)); +} - if (readonly) { - phys_offset |= io_mem_rom.ram_addr; - } - assert(size); +static void register_multipage(MemoryRegionSection *section) +{ + target_phys_addr_t start_addr = section->offset_within_address_space; + ram_addr_t size = section->size; + target_phys_addr_t addr; + uint16_t section_index = phys_section_add(section); - if (phys_offset == io_mem_unassigned.ram_addr) { - region_offset = start_addr; - } - region_offset &= TARGET_PAGE_MASK; - size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK; - end_addr = start_addr + (target_phys_addr_t)size; + assert(size); addr = start_addr; - do { - p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 0); - if (p && p->phys_offset != io_mem_unassigned.ram_addr) { - ram_addr_t orig_memory = p->phys_offset; - target_phys_addr_t start_addr2, end_addr2; - int need_subpage = 0; - MemoryRegion *mr = io_mem_region[orig_memory & ~TARGET_PAGE_MASK]; - - CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, - need_subpage); - if (need_subpage) { - if (!(mr->subpage)) { - subpage = subpage_init((addr & TARGET_PAGE_MASK), - &p->phys_offset, orig_memory, - p->region_offset); - } else { - subpage = container_of(mr, subpage_t, iomem); - } - subpage_register(subpage, start_addr2, end_addr2, phys_offset, - region_offset); - p->region_offset = 0; - } else { - p->phys_offset = phys_offset; - p->region_offset = region_offset; - if (is_ram_rom_romd(phys_offset)) - phys_offset += TARGET_PAGE_SIZE; - } - } else { - p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1); - p->phys_offset = phys_offset; - p->region_offset = region_offset; - if (is_ram_rom_romd(phys_offset)) { - phys_offset += TARGET_PAGE_SIZE; - } else { - target_phys_addr_t start_addr2, end_addr2; - int need_subpage = 0; - - CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, - end_addr2, need_subpage); - - if (need_subpage) { - subpage = subpage_init((addr & TARGET_PAGE_MASK), - &p->phys_offset, - io_mem_unassigned.ram_addr, - addr & TARGET_PAGE_MASK); - subpage_register(subpage, start_addr2, end_addr2, - phys_offset, region_offset); - p->region_offset = 0; - } - } - } - region_offset += TARGET_PAGE_SIZE; - addr += TARGET_PAGE_SIZE; - } while (addr != end_addr); + phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS, + section_index); +} - /* since each CPU stores ram addresses in its TLB cache, we must - reset the modified entries */ - /* XXX: slow ! */ - for(env = first_cpu; env != NULL; env = env->next_cpu) { - tlb_flush(env, 1); +void cpu_register_physical_memory_log(MemoryRegionSection *section, + bool readonly) +{ + MemoryRegionSection now = *section, remain = *section; + + if ((now.offset_within_address_space & ~TARGET_PAGE_MASK) + || (now.size < TARGET_PAGE_SIZE)) { + now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space) + - now.offset_within_address_space, + now.size); + register_subpage(&now); + remain.size -= now.size; + remain.offset_within_address_space += now.size; + remain.offset_within_region += now.size; + } + now = remain; + now.size &= TARGET_PAGE_MASK; + if (now.size) { + register_multipage(&now); + remain.size -= now.size; + remain.offset_within_address_space += now.size; + remain.offset_within_region += now.size; + } + now = remain; + if (now.size) { + register_subpage(&now); } } + void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size) { if (kvm_enabled()) @@ -3257,11 +3332,12 @@ static void check_watchpoint(int offset, int len_mask, int flags) tb_phys_invalidate(tb, -1); if (wp->flags & BP_STOP_BEFORE_ACCESS) { env->exception_index = EXCP_DEBUG; + cpu_loop_exit(env); } else { cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags); tb_gen_code(env, pc, cs_base, cpu_flags, 1); + cpu_resume_from_signal(env, NULL); } - cpu_resume_from_signal(env, NULL); } } else { wp->flags &= ~BP_WATCHPOINT_HIT; @@ -3289,9 +3365,15 @@ static void watch_mem_write(void *opaque, target_phys_addr_t addr, { check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE); switch (size) { - case 1: stb_phys(addr, val); - case 2: stw_phys(addr, val); - case 4: stl_phys(addr, val); + case 1: + stb_phys(addr, val); + break; + case 2: + stw_phys(addr, val); + break; + case 4: + stl_phys(addr, val); + break; default: abort(); } } @@ -3307,14 +3389,17 @@ static uint64_t subpage_read(void *opaque, target_phys_addr_t addr, { subpage_t *mmio = opaque; unsigned int idx = SUBPAGE_IDX(addr); + MemoryRegionSection *section; #if defined(DEBUG_SUBPAGE) printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__, mmio, len, addr, idx); #endif - addr += mmio->region_offset[idx]; - idx = mmio->sub_io_index[idx]; - return io_mem_read(idx, addr, len); + section = &phys_sections[mmio->sub_section[idx]]; + addr += mmio->base; + addr -= section->offset_within_address_space; + addr += section->offset_within_region; + return io_mem_read(section->mr->ram_addr, addr, len); } static void subpage_write(void *opaque, target_phys_addr_t addr, @@ -3322,15 +3407,18 @@ static void subpage_write(void *opaque, target_phys_addr_t addr, { subpage_t *mmio = opaque; unsigned int idx = SUBPAGE_IDX(addr); + MemoryRegionSection *section; #if defined(DEBUG_SUBPAGE) printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %"PRIx64"\n", __func__, mmio, len, addr, idx, value); #endif - addr += mmio->region_offset[idx]; - idx = mmio->sub_io_index[idx]; - io_mem_write(idx, addr, value, len); + section = &phys_sections[mmio->sub_section[idx]]; + addr += mmio->base; + addr -= section->offset_within_address_space; + addr += section->offset_within_region; + io_mem_write(section->mr->ram_addr, addr, value, len); } static const MemoryRegionOps subpage_ops = { @@ -3372,7 +3460,7 @@ static const MemoryRegionOps subpage_ram_ops = { }; static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, - ram_addr_t memory, ram_addr_t region_offset) + uint16_t section) { int idx, eidx; @@ -3384,24 +3472,21 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end, printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__, mmio, start, end, idx, eidx, memory); #endif - if ((memory & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr) { - memory = io_mem_subpage_ram.ram_addr; + if (memory_region_is_ram(phys_sections[section].mr)) { + MemoryRegionSection new_section = phys_sections[section]; + new_section.mr = &io_mem_subpage_ram; + section = phys_section_add(&new_section); } - memory &= IO_MEM_NB_ENTRIES - 1; for (; idx <= eidx; idx++) { - mmio->sub_io_index[idx] = memory; - mmio->region_offset[idx] = region_offset; + mmio->sub_section[idx] = section; } return 0; } -static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, - ram_addr_t orig_memory, - ram_addr_t region_offset) +static subpage_t *subpage_init(target_phys_addr_t base) { subpage_t *mmio; - int subpage_memory; mmio = g_malloc0(sizeof(subpage_t)); @@ -3409,13 +3494,11 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys, memory_region_init_io(&mmio->iomem, &subpage_ops, mmio, "subpage", TARGET_PAGE_SIZE); mmio->iomem.subpage = true; - subpage_memory = mmio->iomem.ram_addr; #if defined(DEBUG_SUBPAGE) printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__, mmio, base, TARGET_PAGE_SIZE, subpage_memory); #endif - *phys = subpage_memory; - subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset); + subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned); return mmio; } @@ -3467,6 +3550,18 @@ void cpu_unregister_io_memory(int io_index) io_mem_used[io_index] = 0; } +static uint16_t dummy_section(MemoryRegion *mr) +{ + MemoryRegionSection section = { + .mr = mr, + .offset_within_address_space = 0, + .offset_within_region = 0, + .size = UINT64_MAX, + }; + + return phys_section_add(§ion); +} + static void io_mem_init(void) { int i; @@ -3488,6 +3583,174 @@ static void io_mem_init(void) "watch", UINT64_MAX); } +static void core_begin(MemoryListener *listener) +{ + destroy_all_mappings(); + phys_sections_clear(); + phys_map.ptr = PHYS_MAP_NODE_NIL; + phys_section_unassigned = dummy_section(&io_mem_unassigned); +} + +static void core_commit(MemoryListener *listener) +{ + CPUState *env; + + /* since each CPU stores ram addresses in its TLB cache, we must + reset the modified entries */ + /* XXX: slow ! */ + for(env = first_cpu; env != NULL; env = env->next_cpu) { + tlb_flush(env, 1); + } +} + +static void core_region_add(MemoryListener *listener, + MemoryRegionSection *section) +{ + cpu_register_physical_memory_log(section, section->readonly); +} + +static void core_region_del(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ + cpu_register_physical_memory_log(section, section->readonly); +} + +static void core_log_start(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_stop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_sync(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void core_log_global_start(MemoryListener *listener) +{ + cpu_physical_memory_set_dirty_tracking(1); +} + +static void core_log_global_stop(MemoryListener *listener) +{ + cpu_physical_memory_set_dirty_tracking(0); +} + +static void core_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void core_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void io_begin(MemoryListener *listener) +{ +} + +static void io_commit(MemoryListener *listener) +{ +} + +static void io_region_add(MemoryListener *listener, + MemoryRegionSection *section) +{ + iorange_init(§ion->mr->iorange, &memory_region_iorange_ops, + section->offset_within_address_space, section->size); + ioport_register(§ion->mr->iorange); +} + +static void io_region_del(MemoryListener *listener, + MemoryRegionSection *section) +{ + isa_unassign_ioport(section->offset_within_address_space, section->size); +} + +static void io_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_start(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_stop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_sync(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + +static void io_log_global_start(MemoryListener *listener) +{ +} + +static void io_log_global_stop(MemoryListener *listener) +{ +} + +static void io_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void io_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static MemoryListener core_memory_listener = { + .begin = core_begin, + .commit = core_commit, + .region_add = core_region_add, + .region_del = core_region_del, + .region_nop = core_region_nop, + .log_start = core_log_start, + .log_stop = core_log_stop, + .log_sync = core_log_sync, + .log_global_start = core_log_global_start, + .log_global_stop = core_log_global_stop, + .eventfd_add = core_eventfd_add, + .eventfd_del = core_eventfd_del, + .priority = 0, +}; + +static MemoryListener io_memory_listener = { + .begin = io_begin, + .commit = io_commit, + .region_add = io_region_add, + .region_del = io_region_del, + .region_nop = io_region_nop, + .log_start = io_log_start, + .log_stop = io_log_stop, + .log_sync = io_log_sync, + .log_global_start = io_log_global_start, + .log_global_stop = io_log_global_stop, + .eventfd_add = io_eventfd_add, + .eventfd_del = io_eventfd_del, + .priority = 0, +}; + static void memory_map_init(void) { system_memory = g_malloc(sizeof(*system_memory)); @@ -3497,6 +3760,9 @@ static void memory_map_init(void) system_io = g_malloc(sizeof(*system_io)); memory_region_init(system_io, "io", 65536); set_system_io_map(system_io); + + memory_listener_register(&core_memory_listener, system_memory); + memory_listener_register(&io_memory_listener, system_io); } MemoryRegion *get_system_memory(void) @@ -3560,22 +3826,22 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, uint8_t *ptr; uint32_t val; target_phys_addr_t page; - ram_addr_t pd; - PhysPageDesc p; + MemoryRegionSection section; while (len > 0) { page = addr & TARGET_PAGE_MASK; l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); if (is_write) { - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { + if (!memory_region_is_ram(section.mr)) { target_phys_addr_t addr1; - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr1 = (addr & ~TARGET_PAGE_MASK) + + section.offset_within_region; /* XXX: could force cpu_single_env to NULL to avoid potential bugs */ if (l >= 4 && ((addr1 & 3) == 0)) { @@ -3594,9 +3860,11 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, io_mem_write(io_index, addr1, val, 1); l = 1; } - } else { + } else if (!section.readonly) { ram_addr_t addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) + | (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); @@ -3610,11 +3878,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, qemu_put_ram_ptr(ptr); } } else { - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { target_phys_addr_t addr1; /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr1 = (addr & ~TARGET_PAGE_MASK) + + section.offset_within_region; if (l >= 4 && ((addr1 & 3) == 0)) { /* 32 bit read access */ val = io_mem_read(io_index, addr1, 4); @@ -3633,7 +3903,8 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf, } } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr(section.mr->ram_addr + + section.offset_within_region); memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l); qemu_put_ram_ptr(ptr); } @@ -3651,22 +3922,22 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr, int l; uint8_t *ptr; target_phys_addr_t page; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; while (len > 0) { page = addr & TARGET_PAGE_MASK; l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* do nothing */ } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); /* ROM/RAM case */ ptr = qemu_get_ram_ptr(addr1); memcpy(ptr, buf, l); @@ -3739,8 +4010,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, target_phys_addr_t todo = 0; int l; target_phys_addr_t page; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; ram_addr_t raddr = RAM_ADDR_MAX; ram_addr_t rlen; void *ret; @@ -3750,10 +4020,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, l = (page + TARGET_PAGE_SIZE) - addr; if (l > len) l = len; - p = phys_page_find(page >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(page >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { + if (!(memory_region_is_ram(section.mr) && !section.readonly)) { if (todo || bounce.buffer) { break; } @@ -3768,7 +4037,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr, return bounce.buffer; } if (!todo) { - raddr = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + raddr = memory_region_get_ram_addr(section.mr) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); } len -= l; @@ -3827,16 +4098,15 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint32_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; val = io_mem_read(io_index, addr, 4); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { @@ -3849,7 +4119,9 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: @@ -3888,16 +4160,15 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint64_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; /* XXX This is broken when device endian != cpu endian. Fix and add "endian" variable check */ @@ -3910,8 +4181,10 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: val = ldq_le_p(ptr); @@ -3957,16 +4230,15 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr, int io_index; uint8_t *ptr; uint64_t val; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if (!is_ram_rom_romd(pd)) { + if (!is_ram_rom_romd(§ion)) { /* I/O case */ - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; val = io_mem_read(io_index, addr, 2); #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { @@ -3979,8 +4251,10 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr, #endif } else { /* RAM case */ - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); switch (endian) { case DEVICE_LITTLE_ENDIAN: val = lduw_le_p(ptr); @@ -4018,18 +4292,23 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val) { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; io_mem_write(io_index, addr, val, 4); } else { - unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + unsigned long addr1 = (memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); ptr = qemu_get_ram_ptr(addr1); stl_p(ptr, val); @@ -4049,15 +4328,18 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val) { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #ifdef TARGET_WORDS_BIGENDIAN io_mem_write(io_index, addr, val >> 32, 4); io_mem_write(io_index, addr + 4, (uint32_t)val, 4); @@ -4066,8 +4348,10 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val) io_mem_write(io_index, addr + 4, val >> 32, 4); #endif } else { - ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) + - (addr & ~TARGET_PAGE_MASK); + ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr) + & TARGET_PAGE_MASK) + + section.offset_within_region) + + (addr & ~TARGET_PAGE_MASK); stq_p(ptr, val); } } @@ -4078,15 +4362,18 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap32(val); @@ -4099,7 +4386,9 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val, io_mem_write(io_index, addr, val, 4); } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK) + + section.offset_within_region + + (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); switch (endian) { @@ -4151,15 +4440,18 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, { int io_index; uint8_t *ptr; - unsigned long pd; - PhysPageDesc p; + MemoryRegionSection section; - p = phys_page_find(addr >> TARGET_PAGE_BITS); - pd = p.phys_offset; + section = phys_page_find(addr >> TARGET_PAGE_BITS); - if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) { - io_index = pd & (IO_MEM_NB_ENTRIES - 1); - addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset; + if (!memory_region_is_ram(section.mr) || section.readonly) { + if (memory_region_is_ram(section.mr)) { + io_index = io_mem_rom.ram_addr; + } else { + io_index = memory_region_get_ram_addr(section.mr) + & (IO_MEM_NB_ENTRIES - 1); + } + addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region; #if defined(TARGET_WORDS_BIGENDIAN) if (endian == DEVICE_LITTLE_ENDIAN) { val = bswap16(val); @@ -4172,7 +4464,8 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val, io_mem_write(io_index, addr, val, 2); } else { unsigned long addr1; - addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK); + addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK) + + section.offset_within_region + (addr & ~TARGET_PAGE_MASK); /* RAM case */ ptr = qemu_get_ram_ptr(addr1); switch (endian) { @@ -4389,7 +4682,7 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr) } pd = env1->tlb_table[mmu_idx][page_index].addr_code & ~TARGET_PAGE_MASK; if (pd != io_mem_ram.ram_addr && pd != io_mem_rom.ram_addr - && !is_romd(pd)) { + && !io_mem_region[pd]->rom_device) { #if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC) cpu_unassigned_access(env1, addr, 0, 1, 0, 4); #else diff --git a/hmp-commands.hx b/hmp-commands.hx index 64b3656d8e..ed88877230 100644 --- a/hmp-commands.hx +++ b/hmp-commands.hx @@ -1355,7 +1355,7 @@ show i8259 (PIC) state @item info pci show emulated PCI device info @item info tlb -show virtual to physical memory mappings (i386, SH4, SPARC, and PPC only) +show virtual to physical memory mappings (i386, SH4, SPARC, PPC, and Xtensa only) @item info mem show the active virtual memory mappings (i386 only) @item info jit @@ -35,6 +35,10 @@ #define MSI_ADDR_DEST_ID_SHIFT 12 #define MSI_ADDR_DEST_ID_MASK 0x00ffff0 +#define SYNC_FROM_VAPIC 0x1 +#define SYNC_TO_VAPIC 0x2 +#define SYNC_ISR_IRR_TO_VAPIC 0x4 + static APICCommonState *local_apics[MAX_APICS + 1]; static void apic_set_irq(APICCommonState *s, int vector_num, int trigger_mode); @@ -78,6 +82,70 @@ static inline int get_bit(uint32_t *tab, int index) return !!(tab[i] & mask); } +/* return -1 if no bit is set */ +static int get_highest_priority_int(uint32_t *tab) +{ + int i; + for (i = 7; i >= 0; i--) { + if (tab[i] != 0) { + return i * 32 + fls_bit(tab[i]); + } + } + return -1; +} + +static void apic_sync_vapic(APICCommonState *s, int sync_type) +{ + VAPICState vapic_state; + size_t length; + off_t start; + int vector; + + if (!s->vapic_paddr) { + return; + } + if (sync_type & SYNC_FROM_VAPIC) { + cpu_physical_memory_rw(s->vapic_paddr, (void *)&vapic_state, + sizeof(vapic_state), 0); + s->tpr = vapic_state.tpr; + } + if (sync_type & (SYNC_TO_VAPIC | SYNC_ISR_IRR_TO_VAPIC)) { + start = offsetof(VAPICState, isr); + length = offsetof(VAPICState, enabled) - offsetof(VAPICState, isr); + + if (sync_type & SYNC_TO_VAPIC) { + assert(qemu_cpu_is_self(s->cpu_env)); + + vapic_state.tpr = s->tpr; + vapic_state.enabled = 1; + start = 0; + length = sizeof(VAPICState); + } + + vector = get_highest_priority_int(s->isr); + if (vector < 0) { + vector = 0; + } + vapic_state.isr = vector & 0xf0; + + vapic_state.zero = 0; + + vector = get_highest_priority_int(s->irr); + if (vector < 0) { + vector = 0; + } + vapic_state.irr = vector & 0xff; + + cpu_physical_memory_write_rom(s->vapic_paddr + start, + ((void *)&vapic_state) + start, length); + } +} + +static void apic_vapic_base_update(APICCommonState *s) +{ + apic_sync_vapic(s, SYNC_TO_VAPIC); +} + static void apic_local_deliver(APICCommonState *s, int vector) { uint32_t lvt = s->lvt[vector]; @@ -239,20 +307,17 @@ static void apic_set_base(APICCommonState *s, uint64_t val) static void apic_set_tpr(APICCommonState *s, uint8_t val) { - s->tpr = (val & 0x0f) << 4; - apic_update_irq(s); + /* Updates from cr8 are ignored while the VAPIC is active */ + if (!s->vapic_paddr) { + s->tpr = val << 4; + apic_update_irq(s); + } } -/* return -1 if no bit is set */ -static int get_highest_priority_int(uint32_t *tab) +static uint8_t apic_get_tpr(APICCommonState *s) { - int i; - for(i = 7; i >= 0; i--) { - if (tab[i] != 0) { - return i * 32 + fls_bit(tab[i]); - } - } - return -1; + apic_sync_vapic(s, SYNC_FROM_VAPIC); + return s->tpr >> 4; } static int apic_get_ppr(APICCommonState *s) @@ -312,6 +377,14 @@ static void apic_update_irq(APICCommonState *s) } } +void apic_poll_irq(DeviceState *d) +{ + APICCommonState *s = APIC_COMMON(d); + + apic_sync_vapic(s, SYNC_FROM_VAPIC); + apic_update_irq(s); +} + static void apic_set_irq(APICCommonState *s, int vector_num, int trigger_mode) { apic_report_irq_delivered(!get_bit(s->irr, vector_num)); @@ -321,6 +394,16 @@ static void apic_set_irq(APICCommonState *s, int vector_num, int trigger_mode) set_bit(s->tmr, vector_num); else reset_bit(s->tmr, vector_num); + if (s->vapic_paddr) { + apic_sync_vapic(s, SYNC_ISR_IRR_TO_VAPIC); + /* + * The vcpu thread needs to see the new IRR before we pull its current + * TPR value. That way, if we miss a lowering of the TRP, the guest + * has the chance to notice the new IRR and poll for IRQs on its own. + */ + smp_wmb(); + apic_sync_vapic(s, SYNC_FROM_VAPIC); + } apic_update_irq(s); } @@ -334,6 +417,7 @@ static void apic_eoi(APICCommonState *s) if (!(s->spurious_vec & APIC_SV_DIRECTED_IO) && get_bit(s->tmr, isrv)) { ioapic_eoi_broadcast(isrv); } + apic_sync_vapic(s, SYNC_FROM_VAPIC | SYNC_TO_VAPIC); apic_update_irq(s); } @@ -471,15 +555,19 @@ int apic_get_interrupt(DeviceState *d) if (!(s->spurious_vec & APIC_SV_ENABLE)) return -1; + apic_sync_vapic(s, SYNC_FROM_VAPIC); intno = apic_irq_pending(s); if (intno == 0) { + apic_sync_vapic(s, SYNC_TO_VAPIC); return -1; } else if (intno < 0) { + apic_sync_vapic(s, SYNC_TO_VAPIC); return s->spurious_vec & 0xff; } reset_bit(s->irr, intno); set_bit(s->isr, intno); + apic_sync_vapic(s, SYNC_TO_VAPIC); apic_update_irq(s); return intno; } @@ -576,6 +664,10 @@ static uint32_t apic_mem_readl(void *opaque, target_phys_addr_t addr) val = 0x11 | ((APIC_LVT_NB - 1) << 16); /* version 0x11 */ break; case 0x08: + apic_sync_vapic(s, SYNC_FROM_VAPIC); + if (apic_report_tpr_access) { + cpu_report_tpr_access(s->cpu_env, TPR_ACCESS_READ); + } val = s->tpr; break; case 0x09: @@ -675,7 +767,11 @@ static void apic_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val) case 0x03: break; case 0x08: + if (apic_report_tpr_access) { + cpu_report_tpr_access(s->cpu_env, TPR_ACCESS_WRITE); + } s->tpr = val; + apic_sync_vapic(s, SYNC_TO_VAPIC); apic_update_irq(s); break; case 0x09: @@ -737,6 +833,11 @@ static void apic_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val) } } +static void apic_pre_save(APICCommonState *s) +{ + apic_sync_vapic(s, SYNC_FROM_VAPIC); +} + static void apic_post_load(APICCommonState *s) { if (s->timer_expiry != -1) { @@ -770,7 +871,10 @@ static void apic_class_init(ObjectClass *klass, void *data) k->init = apic_init; k->set_base = apic_set_base; k->set_tpr = apic_set_tpr; + k->get_tpr = apic_get_tpr; + k->vapic_base_update = apic_vapic_base_update; k->external_nmi = apic_external_nmi; + k->pre_save = apic_pre_save; k->post_load = apic_post_load; } @@ -18,6 +18,8 @@ void cpu_set_apic_tpr(DeviceState *s, uint8_t val); uint8_t cpu_get_apic_tpr(DeviceState *s); void apic_init_reset(DeviceState *s); void apic_sipi(DeviceState *s); +void apic_handle_tpr_access_report(DeviceState *d, target_ulong ip, + TPRAccess access); /* pc.c */ int cpu_is_bsp(CPUState *env); diff --git a/hw/apic_common.c b/hw/apic_common.c index c91f7d5391..60b82596e7 100644 --- a/hw/apic_common.c +++ b/hw/apic_common.c @@ -20,8 +20,10 @@ #include "apic.h" #include "apic_internal.h" #include "trace.h" +#include "kvm.h" static int apic_irq_delivered; +bool apic_report_tpr_access; void cpu_set_apic_base(DeviceState *d, uint64_t val) { @@ -63,9 +65,45 @@ void cpu_set_apic_tpr(DeviceState *d, uint8_t val) uint8_t cpu_get_apic_tpr(DeviceState *d) { + APICCommonState *s; + APICCommonClass *info; + + if (!d) { + return 0; + } + + s = APIC_COMMON(d); + info = APIC_COMMON_GET_CLASS(s); + + return info->get_tpr(s); +} + +void apic_enable_tpr_access_reporting(DeviceState *d, bool enable) +{ APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d); + APICCommonClass *info = APIC_COMMON_GET_CLASS(s); - return s ? s->tpr >> 4 : 0; + apic_report_tpr_access = enable; + if (info->enable_tpr_reporting) { + info->enable_tpr_reporting(s, enable); + } +} + +void apic_enable_vapic(DeviceState *d, target_phys_addr_t paddr) +{ + APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d); + APICCommonClass *info = APIC_COMMON_GET_CLASS(s); + + s->vapic_paddr = paddr; + info->vapic_base_update(s); +} + +void apic_handle_tpr_access_report(DeviceState *d, target_ulong ip, + TPRAccess access) +{ + APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d); + + vapic_report_tpr_access(s->vapic, s->cpu_env, ip, access); } void apic_report_irq_delivered(int delivered) @@ -166,12 +204,16 @@ void apic_init_reset(DeviceState *d) static void apic_reset_common(DeviceState *d) { APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d); + APICCommonClass *info = APIC_COMMON_GET_CLASS(s); bool bsp; bsp = cpu_is_bsp(s->cpu_env); s->apicbase = 0xfee00000 | (bsp ? MSR_IA32_APICBASE_BSP : 0) | MSR_IA32_APICBASE_ENABLE; + s->vapic_paddr = 0; + info->vapic_base_update(s); + apic_init_reset(d); if (bsp) { @@ -234,6 +276,7 @@ static int apic_init_common(SysBusDevice *dev) { APICCommonState *s = APIC_COMMON(dev); APICCommonClass *info; + static DeviceState *vapic; static int apic_no; if (apic_no >= MAX_APICS) { @@ -244,10 +287,29 @@ static int apic_init_common(SysBusDevice *dev) info = APIC_COMMON_GET_CLASS(s); info->init(s); - sysbus_init_mmio(&s->busdev, &s->io_memory); + sysbus_init_mmio(dev, &s->io_memory); + + if (!vapic && s->vapic_control & VAPIC_ENABLE_MASK) { + vapic = sysbus_create_simple("kvmvapic", -1, NULL); + } + s->vapic = vapic; + if (apic_report_tpr_access && info->enable_tpr_reporting) { + info->enable_tpr_reporting(s, true); + } + return 0; } +static void apic_dispatch_pre_save(void *opaque) +{ + APICCommonState *s = APIC_COMMON(opaque); + APICCommonClass *info = APIC_COMMON_GET_CLASS(s); + + if (info->pre_save) { + info->pre_save(s); + } +} + static int apic_dispatch_post_load(void *opaque, int version_id) { APICCommonState *s = APIC_COMMON(opaque); @@ -265,6 +327,7 @@ static const VMStateDescription vmstate_apic_common = { .minimum_version_id = 3, .minimum_version_id_old = 1, .load_state_old = apic_load_old, + .pre_save = apic_dispatch_pre_save, .post_load = apic_dispatch_post_load, .fields = (VMStateField[]) { VMSTATE_UINT32(apicbase, APICCommonState), @@ -294,6 +357,8 @@ static const VMStateDescription vmstate_apic_common = { static Property apic_properties_common[] = { DEFINE_PROP_UINT8("id", APICCommonState, id, -1), DEFINE_PROP_PTR("cpu_env", APICCommonState, cpu_env), + DEFINE_PROP_BIT("vapic", APICCommonState, vapic_control, VAPIC_ENABLE_BIT, + true), DEFINE_PROP_END_OF_LIST(), }; diff --git a/hw/apic_internal.h b/hw/apic_internal.h index 0cab010717..60a6a8bdae 100644 --- a/hw/apic_internal.h +++ b/hw/apic_internal.h @@ -61,6 +61,9 @@ #define APIC_SV_DIRECTED_IO (1<<12) #define APIC_SV_ENABLE (1<<8) +#define VAPIC_ENABLE_BIT 0 +#define VAPIC_ENABLE_MASK (1 << VAPIC_ENABLE_BIT) + #define MAX_APICS 255 #define MSI_SPACE_SIZE 0x100000 @@ -82,7 +85,11 @@ typedef struct APICCommonClass void (*init)(APICCommonState *s); void (*set_base)(APICCommonState *s, uint64_t val); void (*set_tpr)(APICCommonState *s, uint8_t val); + uint8_t (*get_tpr)(APICCommonState *s); + void (*enable_tpr_reporting)(APICCommonState *s, bool enable); + void (*vapic_base_update)(APICCommonState *s); void (*external_nmi)(APICCommonState *s); + void (*pre_save)(APICCommonState *s); void (*post_load)(APICCommonState *s); } APICCommonClass; @@ -114,9 +121,29 @@ struct APICCommonState { int64_t timer_expiry; int sipi_vector; int wait_for_sipi; + + uint32_t vapic_control; + DeviceState *vapic; + target_phys_addr_t vapic_paddr; /* note: persistence via kvmvapic */ }; +typedef struct VAPICState { + uint8_t tpr; + uint8_t isr; + uint8_t zero; + uint8_t irr; + uint8_t enabled; +} QEMU_PACKED VAPICState; + +extern bool apic_report_tpr_access; + void apic_report_irq_delivered(int delivered); bool apic_next_timer(APICCommonState *s, int64_t current_time); +void apic_enable_tpr_access_reporting(DeviceState *d, bool enable); +void apic_enable_vapic(DeviceState *d, target_phys_addr_t paddr); +void apic_poll_irq(DeviceState *d); + +void vapic_report_tpr_access(DeviceState *dev, void *cpu, target_ulong ip, + TPRAccess access); #endif /* !QEMU_APIC_INTERNAL_H */ diff --git a/hw/kvm/apic.c b/hw/kvm/apic.c index 5bb0a4b9fd..9ca68f81aa 100644 --- a/hw/kvm/apic.c +++ b/hw/kvm/apic.c @@ -92,6 +92,35 @@ static void kvm_apic_set_tpr(APICCommonState *s, uint8_t val) s->tpr = (val & 0x0f) << 4; } +static uint8_t kvm_apic_get_tpr(APICCommonState *s) +{ + return s->tpr >> 4; +} + +static void kvm_apic_enable_tpr_reporting(APICCommonState *s, bool enable) +{ + struct kvm_tpr_access_ctl ctl = { + .enabled = enable + }; + + kvm_vcpu_ioctl(s->cpu_env, KVM_TPR_ACCESS_REPORTING, &ctl); +} + +static void kvm_apic_vapic_base_update(APICCommonState *s) +{ + struct kvm_vapic_addr vapid_addr = { + .vapic_addr = s->vapic_paddr, + }; + int ret; + + ret = kvm_vcpu_ioctl(s->cpu_env, KVM_SET_VAPIC_ADDR, &vapid_addr); + if (ret < 0) { + fprintf(stderr, "KVM: setting VAPIC address failed (%s)\n", + strerror(-ret)); + abort(); + } +} + static void do_inject_external_nmi(void *data) { APICCommonState *s = data; @@ -129,6 +158,9 @@ static void kvm_apic_class_init(ObjectClass *klass, void *data) k->init = kvm_apic_init; k->set_base = kvm_apic_set_base; k->set_tpr = kvm_apic_set_tpr; + k->get_tpr = kvm_apic_get_tpr; + k->enable_tpr_reporting = kvm_apic_enable_tpr_reporting; + k->vapic_base_update = kvm_apic_vapic_base_update; k->external_nmi = kvm_apic_external_nmi; } diff --git a/hw/kvmvapic.c b/hw/kvmvapic.c new file mode 100644 index 0000000000..36ccfbcdbd --- /dev/null +++ b/hw/kvmvapic.c @@ -0,0 +1,805 @@ +/* + * TPR optimization for 32-bit Windows guests (XP and Server 2003) + * + * Copyright (C) 2007-2008 Qumranet Technologies + * Copyright (C) 2012 Jan Kiszka, Siemens AG + * + * This work is licensed under the terms of the GNU GPL version 2, or + * (at your option) any later version. See the COPYING file in the + * top-level directory. + */ +#include "sysemu.h" +#include "cpus.h" +#include "kvm.h" +#include "apic_internal.h" + +#define APIC_DEFAULT_ADDRESS 0xfee00000 + +#define VAPIC_IO_PORT 0x7e + +#define VAPIC_CPU_SHIFT 7 + +#define ROM_BLOCK_SIZE 512 +#define ROM_BLOCK_MASK (~(ROM_BLOCK_SIZE - 1)) + +typedef enum VAPICMode { + VAPIC_INACTIVE = 0, + VAPIC_ACTIVE = 1, + VAPIC_STANDBY = 2, +} VAPICMode; + +typedef struct VAPICHandlers { + uint32_t set_tpr; + uint32_t set_tpr_eax; + uint32_t get_tpr[8]; + uint32_t get_tpr_stack; +} QEMU_PACKED VAPICHandlers; + +typedef struct GuestROMState { + char signature[8]; + uint32_t vaddr; + uint32_t fixup_start; + uint32_t fixup_end; + uint32_t vapic_vaddr; + uint32_t vapic_size; + uint32_t vcpu_shift; + uint32_t real_tpr_addr; + VAPICHandlers up; + VAPICHandlers mp; +} QEMU_PACKED GuestROMState; + +typedef struct VAPICROMState { + SysBusDevice busdev; + MemoryRegion io; + MemoryRegion rom; + uint32_t state; + uint32_t rom_state_paddr; + uint32_t rom_state_vaddr; + uint32_t vapic_paddr; + uint32_t real_tpr_addr; + GuestROMState rom_state; + size_t rom_size; + bool rom_mapped_writable; +} VAPICROMState; + +#define TPR_INSTR_ABS_MODRM 0x1 +#define TPR_INSTR_MATCH_MODRM_REG 0x2 + +typedef struct TPRInstruction { + uint8_t opcode; + uint8_t modrm_reg; + unsigned int flags; + TPRAccess access; + size_t length; + off_t addr_offset; +} TPRInstruction; + +/* must be sorted by length, shortest first */ +static const TPRInstruction tpr_instr[] = { + { /* mov abs to eax */ + .opcode = 0xa1, + .access = TPR_ACCESS_READ, + .length = 5, + .addr_offset = 1, + }, + { /* mov eax to abs */ + .opcode = 0xa3, + .access = TPR_ACCESS_WRITE, + .length = 5, + .addr_offset = 1, + }, + { /* mov r32 to r/m32 */ + .opcode = 0x89, + .flags = TPR_INSTR_ABS_MODRM, + .access = TPR_ACCESS_WRITE, + .length = 6, + .addr_offset = 2, + }, + { /* mov r/m32 to r32 */ + .opcode = 0x8b, + .flags = TPR_INSTR_ABS_MODRM, + .access = TPR_ACCESS_READ, + .length = 6, + .addr_offset = 2, + }, + { /* push r/m32 */ + .opcode = 0xff, + .modrm_reg = 6, + .flags = TPR_INSTR_ABS_MODRM | TPR_INSTR_MATCH_MODRM_REG, + .access = TPR_ACCESS_READ, + .length = 6, + .addr_offset = 2, + }, + { /* mov imm32, r/m32 (c7/0) */ + .opcode = 0xc7, + .modrm_reg = 0, + .flags = TPR_INSTR_ABS_MODRM | TPR_INSTR_MATCH_MODRM_REG, + .access = TPR_ACCESS_WRITE, + .length = 10, + .addr_offset = 2, + }, +}; + +static void read_guest_rom_state(VAPICROMState *s) +{ + cpu_physical_memory_rw(s->rom_state_paddr, (void *)&s->rom_state, + sizeof(GuestROMState), 0); +} + +static void write_guest_rom_state(VAPICROMState *s) +{ + cpu_physical_memory_rw(s->rom_state_paddr, (void *)&s->rom_state, + sizeof(GuestROMState), 1); +} + +static void update_guest_rom_state(VAPICROMState *s) +{ + read_guest_rom_state(s); + + s->rom_state.real_tpr_addr = cpu_to_le32(s->real_tpr_addr); + s->rom_state.vcpu_shift = cpu_to_le32(VAPIC_CPU_SHIFT); + + write_guest_rom_state(s); +} + +static int find_real_tpr_addr(VAPICROMState *s, CPUState *env) +{ + target_phys_addr_t paddr; + target_ulong addr; + + if (s->state == VAPIC_ACTIVE) { + return 0; + } + /* + * If there is no prior TPR access instruction we could analyze (which is + * the case after resume from hibernation), we need to scan the possible + * virtual address space for the APIC mapping. + */ + for (addr = 0xfffff000; addr >= 0x80000000; addr -= TARGET_PAGE_SIZE) { + paddr = cpu_get_phys_page_debug(env, addr); + if (paddr != APIC_DEFAULT_ADDRESS) { + continue; + } + s->real_tpr_addr = addr + 0x80; + update_guest_rom_state(s); + return 0; + } + return -1; +} + +static uint8_t modrm_reg(uint8_t modrm) +{ + return (modrm >> 3) & 7; +} + +static bool is_abs_modrm(uint8_t modrm) +{ + return (modrm & 0xc7) == 0x05; +} + +static bool opcode_matches(uint8_t *opcode, const TPRInstruction *instr) +{ + return opcode[0] == instr->opcode && + (!(instr->flags & TPR_INSTR_ABS_MODRM) || is_abs_modrm(opcode[1])) && + (!(instr->flags & TPR_INSTR_MATCH_MODRM_REG) || + modrm_reg(opcode[1]) == instr->modrm_reg); +} + +static int evaluate_tpr_instruction(VAPICROMState *s, CPUState *env, + target_ulong *pip, TPRAccess access) +{ + const TPRInstruction *instr; + target_ulong ip = *pip; + uint8_t opcode[2]; + uint32_t real_tpr_addr; + int i; + + if ((ip & 0xf0000000ULL) != 0x80000000ULL && + (ip & 0xf0000000ULL) != 0xe0000000ULL) { + return -1; + } + + /* + * Early Windows 2003 SMP initialization contains a + * + * mov imm32, r/m32 + * + * instruction that is patched by TPR optimization. The problem is that + * RSP, used by the patched instruction, is zero, so the guest gets a + * double fault and dies. + */ + if (env->regs[R_ESP] == 0) { + return -1; + } + + if (kvm_enabled() && !kvm_irqchip_in_kernel()) { + /* + * KVM without kernel-based TPR access reporting will pass an IP that + * points after the accessing instruction. So we need to look backward + * to find the reason. + */ + for (i = 0; i < ARRAY_SIZE(tpr_instr); i++) { + instr = &tpr_instr[i]; + if (instr->access != access) { + continue; + } + if (cpu_memory_rw_debug(env, ip - instr->length, opcode, + sizeof(opcode), 0) < 0) { + return -1; + } + if (opcode_matches(opcode, instr)) { + ip -= instr->length; + goto instruction_ok; + } + } + return -1; + } else { + if (cpu_memory_rw_debug(env, ip, opcode, sizeof(opcode), 0) < 0) { + return -1; + } + for (i = 0; i < ARRAY_SIZE(tpr_instr); i++) { + instr = &tpr_instr[i]; + if (opcode_matches(opcode, instr)) { + goto instruction_ok; + } + } + return -1; + } + +instruction_ok: + /* + * Grab the virtual TPR address from the instruction + * and update the cached values. + */ + if (cpu_memory_rw_debug(env, ip + instr->addr_offset, + (void *)&real_tpr_addr, + sizeof(real_tpr_addr), 0) < 0) { + return -1; + } + real_tpr_addr = le32_to_cpu(real_tpr_addr); + if ((real_tpr_addr & 0xfff) != 0x80) { + return -1; + } + s->real_tpr_addr = real_tpr_addr; + update_guest_rom_state(s); + + *pip = ip; + return 0; +} + +static int update_rom_mapping(VAPICROMState *s, CPUState *env, target_ulong ip) +{ + target_phys_addr_t paddr; + uint32_t rom_state_vaddr; + uint32_t pos, patch, offset; + + /* nothing to do if already activated */ + if (s->state == VAPIC_ACTIVE) { + return 0; + } + + /* bail out if ROM init code was not executed (missing ROM?) */ + if (s->state == VAPIC_INACTIVE) { + return -1; + } + + /* find out virtual address of the ROM */ + rom_state_vaddr = s->rom_state_paddr + (ip & 0xf0000000); + paddr = cpu_get_phys_page_debug(env, rom_state_vaddr); + if (paddr == -1) { + return -1; + } + paddr += rom_state_vaddr & ~TARGET_PAGE_MASK; + if (paddr != s->rom_state_paddr) { + return -1; + } + read_guest_rom_state(s); + if (memcmp(s->rom_state.signature, "kvm aPiC", 8) != 0) { + return -1; + } + s->rom_state_vaddr = rom_state_vaddr; + + /* fixup addresses in ROM if needed */ + if (rom_state_vaddr == le32_to_cpu(s->rom_state.vaddr)) { + return 0; + } + for (pos = le32_to_cpu(s->rom_state.fixup_start); + pos < le32_to_cpu(s->rom_state.fixup_end); + pos += 4) { + cpu_physical_memory_rw(paddr + pos - s->rom_state.vaddr, + (void *)&offset, sizeof(offset), 0); + offset = le32_to_cpu(offset); + cpu_physical_memory_rw(paddr + offset, (void *)&patch, + sizeof(patch), 0); + patch = le32_to_cpu(patch); + patch += rom_state_vaddr - le32_to_cpu(s->rom_state.vaddr); + patch = cpu_to_le32(patch); + cpu_physical_memory_rw(paddr + offset, (void *)&patch, + sizeof(patch), 1); + } + read_guest_rom_state(s); + s->vapic_paddr = paddr + le32_to_cpu(s->rom_state.vapic_vaddr) - + le32_to_cpu(s->rom_state.vaddr); + + return 0; +} + +/* + * Tries to read the unique processor number from the Kernel Processor Control + * Region (KPCR) of 32-bit Windows XP and Server 2003. Returns -1 if the KPCR + * cannot be accessed or is considered invalid. This also ensures that we are + * not patching the wrong guest. + */ +static int get_kpcr_number(CPUState *env) +{ + struct kpcr { + uint8_t fill1[0x1c]; + uint32_t self; + uint8_t fill2[0x31]; + uint8_t number; + } QEMU_PACKED kpcr; + + if (cpu_memory_rw_debug(env, env->segs[R_FS].base, + (void *)&kpcr, sizeof(kpcr), 0) < 0 || + kpcr.self != env->segs[R_FS].base) { + return -1; + } + return kpcr.number; +} + +static int vapic_enable(VAPICROMState *s, CPUState *env) +{ + int cpu_number = get_kpcr_number(env); + target_phys_addr_t vapic_paddr; + static const uint8_t enabled = 1; + + if (cpu_number < 0) { + return -1; + } + vapic_paddr = s->vapic_paddr + + (((target_phys_addr_t)cpu_number) << VAPIC_CPU_SHIFT); + cpu_physical_memory_rw(vapic_paddr + offsetof(VAPICState, enabled), + (void *)&enabled, sizeof(enabled), 1); + apic_enable_vapic(env->apic_state, vapic_paddr); + + s->state = VAPIC_ACTIVE; + + return 0; +} + +static void patch_byte(CPUState *env, target_ulong addr, uint8_t byte) +{ + cpu_memory_rw_debug(env, addr, &byte, 1, 1); +} + +static void patch_call(VAPICROMState *s, CPUState *env, target_ulong ip, + uint32_t target) +{ + uint32_t offset; + + offset = cpu_to_le32(target - ip - 5); + patch_byte(env, ip, 0xe8); /* call near */ + cpu_memory_rw_debug(env, ip + 1, (void *)&offset, sizeof(offset), 1); +} + +static void patch_instruction(VAPICROMState *s, CPUState *env, target_ulong ip) +{ + target_phys_addr_t paddr; + VAPICHandlers *handlers; + uint8_t opcode[2]; + uint32_t imm32; + + if (smp_cpus == 1) { + handlers = &s->rom_state.up; + } else { + handlers = &s->rom_state.mp; + } + + pause_all_vcpus(); + + cpu_memory_rw_debug(env, ip, opcode, sizeof(opcode), 0); + + switch (opcode[0]) { + case 0x89: /* mov r32 to r/m32 */ + patch_byte(env, ip, 0x50 + modrm_reg(opcode[1])); /* push reg */ + patch_call(s, env, ip + 1, handlers->set_tpr); + break; + case 0x8b: /* mov r/m32 to r32 */ + patch_byte(env, ip, 0x90); + patch_call(s, env, ip + 1, handlers->get_tpr[modrm_reg(opcode[1])]); + break; + case 0xa1: /* mov abs to eax */ + patch_call(s, env, ip, handlers->get_tpr[0]); + break; + case 0xa3: /* mov eax to abs */ + patch_call(s, env, ip, handlers->set_tpr_eax); + break; + case 0xc7: /* mov imm32, r/m32 (c7/0) */ + patch_byte(env, ip, 0x68); /* push imm32 */ + cpu_memory_rw_debug(env, ip + 6, (void *)&imm32, sizeof(imm32), 0); + cpu_memory_rw_debug(env, ip + 1, (void *)&imm32, sizeof(imm32), 1); + patch_call(s, env, ip + 5, handlers->set_tpr); + break; + case 0xff: /* push r/m32 */ + patch_byte(env, ip, 0x50); /* push eax */ + patch_call(s, env, ip + 1, handlers->get_tpr_stack); + break; + default: + abort(); + } + + resume_all_vcpus(); + + paddr = cpu_get_phys_page_debug(env, ip); + paddr += ip & ~TARGET_PAGE_MASK; + tb_invalidate_phys_page_range(paddr, paddr + 1, 1); +} + +void vapic_report_tpr_access(DeviceState *dev, void *cpu, target_ulong ip, + TPRAccess access) +{ + VAPICROMState *s = DO_UPCAST(VAPICROMState, busdev.qdev, dev); + CPUState *env = cpu; + + cpu_synchronize_state(env); + + if (evaluate_tpr_instruction(s, env, &ip, access) < 0) { + if (s->state == VAPIC_ACTIVE) { + vapic_enable(s, env); + } + return; + } + if (update_rom_mapping(s, env, ip) < 0) { + return; + } + if (vapic_enable(s, env) < 0) { + return; + } + patch_instruction(s, env, ip); +} + +typedef struct VAPICEnableTPRReporting { + DeviceState *apic; + bool enable; +} VAPICEnableTPRReporting; + +static void vapic_do_enable_tpr_reporting(void *data) +{ + VAPICEnableTPRReporting *info = data; + + apic_enable_tpr_access_reporting(info->apic, info->enable); +} + +static void vapic_enable_tpr_reporting(bool enable) +{ + VAPICEnableTPRReporting info = { + .enable = enable, + }; + CPUState *env; + + for (env = first_cpu; env != NULL; env = env->next_cpu) { + info.apic = env->apic_state; + run_on_cpu(env, vapic_do_enable_tpr_reporting, &info); + } +} + +static void vapic_reset(DeviceState *dev) +{ + VAPICROMState *s = DO_UPCAST(VAPICROMState, busdev.qdev, dev); + + if (s->state == VAPIC_ACTIVE) { + s->state = VAPIC_STANDBY; + } + vapic_enable_tpr_reporting(false); +} + +/* + * Set the IRQ polling hypercalls to the supported variant: + * - vmcall if using KVM in-kernel irqchip + * - 32-bit VAPIC port write otherwise + */ +static int patch_hypercalls(VAPICROMState *s) +{ + target_phys_addr_t rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK; + static const uint8_t vmcall_pattern[] = { /* vmcall */ + 0xb8, 0x1, 0, 0, 0, 0xf, 0x1, 0xc1 + }; + static const uint8_t outl_pattern[] = { /* nop; outl %eax,0x7e */ + 0xb8, 0x1, 0, 0, 0, 0x90, 0xe7, 0x7e + }; + uint8_t alternates[2]; + const uint8_t *pattern; + const uint8_t *patch; + int patches = 0; + off_t pos; + uint8_t *rom; + + rom = g_malloc(s->rom_size); + cpu_physical_memory_rw(rom_paddr, rom, s->rom_size, 0); + + for (pos = 0; pos < s->rom_size - sizeof(vmcall_pattern); pos++) { + if (kvm_irqchip_in_kernel()) { + pattern = outl_pattern; + alternates[0] = outl_pattern[7]; + alternates[1] = outl_pattern[7]; + patch = &vmcall_pattern[5]; + } else { + pattern = vmcall_pattern; + alternates[0] = vmcall_pattern[7]; + alternates[1] = 0xd9; /* AMD's VMMCALL */ + patch = &outl_pattern[5]; + } + if (memcmp(rom + pos, pattern, 7) == 0 && + (rom[pos + 7] == alternates[0] || rom[pos + 7] == alternates[1])) { + cpu_physical_memory_rw(rom_paddr + pos + 5, (uint8_t *)patch, + 3, 1); + /* + * Don't flush the tb here. Under ordinary conditions, the patched + * calls are miles away from the current IP. Under malicious + * conditions, the guest could trick us to crash. + */ + } + } + + g_free(rom); + + if (patches != 0 && patches != 2) { + return -1; + } + + return 0; +} + +/* + * For TCG mode or the time KVM honors read-only memory regions, we need to + * enable write access to the option ROM so that variables can be updated by + * the guest. + */ +static void vapic_map_rom_writable(VAPICROMState *s) +{ + target_phys_addr_t rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK; + MemoryRegionSection section; + MemoryRegion *as; + size_t rom_size; + uint8_t *ram; + + as = sysbus_address_space(&s->busdev); + + if (s->rom_mapped_writable) { + memory_region_del_subregion(as, &s->rom); + memory_region_destroy(&s->rom); + } + + /* grab RAM memory region (region @rom_paddr may still be pc.rom) */ + section = memory_region_find(as, 0, 1); + + /* read ROM size from RAM region */ + ram = memory_region_get_ram_ptr(section.mr); + rom_size = ram[rom_paddr + 2] * ROM_BLOCK_SIZE; + s->rom_size = rom_size; + + /* We need to round up to avoid creating subpages + * from which we cannot run code. */ + rom_size = TARGET_PAGE_ALIGN(rom_size); + + memory_region_init_alias(&s->rom, "kvmvapic-rom", section.mr, rom_paddr, + rom_size); + memory_region_add_subregion_overlap(as, rom_paddr, &s->rom, 1000); + s->rom_mapped_writable = true; +} + +static int vapic_prepare(VAPICROMState *s) +{ + vapic_map_rom_writable(s); + + if (patch_hypercalls(s) < 0) { + return -1; + } + + vapic_enable_tpr_reporting(true); + + return 0; +} + +static void vapic_write(void *opaque, target_phys_addr_t addr, uint64_t data, + unsigned int size) +{ + CPUState *env = cpu_single_env; + target_phys_addr_t rom_paddr; + VAPICROMState *s = opaque; + + cpu_synchronize_state(env); + + /* + * The VAPIC supports two PIO-based hypercalls, both via port 0x7E. + * o 16-bit write access: + * Reports the option ROM initialization to the hypervisor. Written + * value is the offset of the state structure in the ROM. + * o 8-bit write access: + * Reactivates the VAPIC after a guest hibernation, i.e. after the + * option ROM content has been re-initialized by a guest power cycle. + * o 32-bit write access: + * Poll for pending IRQs, considering the current VAPIC state. + */ + switch (size) { + case 2: + if (s->state == VAPIC_INACTIVE) { + rom_paddr = (env->segs[R_CS].base + env->eip) & ROM_BLOCK_MASK; + s->rom_state_paddr = rom_paddr + data; + + s->state = VAPIC_STANDBY; + } + if (vapic_prepare(s) < 0) { + s->state = VAPIC_INACTIVE; + break; + } + break; + case 1: + if (kvm_enabled()) { + /* + * Disable triggering instruction in ROM by writing a NOP. + * + * We cannot do this in TCG mode as the reported IP is not + * accurate. + */ + pause_all_vcpus(); + patch_byte(env, env->eip - 2, 0x66); + patch_byte(env, env->eip - 1, 0x90); + resume_all_vcpus(); + } + + if (s->state == VAPIC_ACTIVE) { + break; + } + if (update_rom_mapping(s, env, env->eip) < 0) { + break; + } + if (find_real_tpr_addr(s, env) < 0) { + break; + } + vapic_enable(s, env); + break; + default: + case 4: + if (!kvm_irqchip_in_kernel()) { + apic_poll_irq(env->apic_state); + } + break; + } +} + +static const MemoryRegionOps vapic_ops = { + .write = vapic_write, + .endianness = DEVICE_NATIVE_ENDIAN, +}; + +static int vapic_init(SysBusDevice *dev) +{ + VAPICROMState *s = FROM_SYSBUS(VAPICROMState, dev); + + memory_region_init_io(&s->io, &vapic_ops, s, "kvmvapic", 2); + sysbus_add_io(dev, VAPIC_IO_PORT, &s->io); + sysbus_init_ioports(dev, VAPIC_IO_PORT, 2); + + option_rom[nb_option_roms].name = "kvmvapic.bin"; + option_rom[nb_option_roms].bootindex = -1; + nb_option_roms++; + + return 0; +} + +static void do_vapic_enable(void *data) +{ + VAPICROMState *s = data; + + vapic_enable(s, first_cpu); +} + +static int vapic_post_load(void *opaque, int version_id) +{ + VAPICROMState *s = opaque; + uint8_t *zero; + + /* + * The old implementation of qemu-kvm did not provide the state + * VAPIC_STANDBY. Reconstruct it. + */ + if (s->state == VAPIC_INACTIVE && s->rom_state_paddr != 0) { + s->state = VAPIC_STANDBY; + } + + if (s->state != VAPIC_INACTIVE) { + if (vapic_prepare(s) < 0) { + return -1; + } + } + if (s->state == VAPIC_ACTIVE) { + if (smp_cpus == 1) { + run_on_cpu(first_cpu, do_vapic_enable, s); + } else { + zero = g_malloc0(s->rom_state.vapic_size); + cpu_physical_memory_rw(s->vapic_paddr, zero, + s->rom_state.vapic_size, 1); + g_free(zero); + } + } + + return 0; +} + +static const VMStateDescription vmstate_handlers = { + .name = "kvmvapic-handlers", + .version_id = 1, + .minimum_version_id = 1, + .minimum_version_id_old = 1, + .fields = (VMStateField[]) { + VMSTATE_UINT32(set_tpr, VAPICHandlers), + VMSTATE_UINT32(set_tpr_eax, VAPICHandlers), + VMSTATE_UINT32_ARRAY(get_tpr, VAPICHandlers, 8), + VMSTATE_UINT32(get_tpr_stack, VAPICHandlers), + VMSTATE_END_OF_LIST() + } +}; + +static const VMStateDescription vmstate_guest_rom = { + .name = "kvmvapic-guest-rom", + .version_id = 1, + .minimum_version_id = 1, + .minimum_version_id_old = 1, + .fields = (VMStateField[]) { + VMSTATE_UNUSED(8), /* signature */ + VMSTATE_UINT32(vaddr, GuestROMState), + VMSTATE_UINT32(fixup_start, GuestROMState), + VMSTATE_UINT32(fixup_end, GuestROMState), + VMSTATE_UINT32(vapic_vaddr, GuestROMState), + VMSTATE_UINT32(vapic_size, GuestROMState), + VMSTATE_UINT32(vcpu_shift, GuestROMState), + VMSTATE_UINT32(real_tpr_addr, GuestROMState), + VMSTATE_STRUCT(up, GuestROMState, 0, vmstate_handlers, VAPICHandlers), + VMSTATE_STRUCT(mp, GuestROMState, 0, vmstate_handlers, VAPICHandlers), + VMSTATE_END_OF_LIST() + } +}; + +static const VMStateDescription vmstate_vapic = { + .name = "kvm-tpr-opt", /* compatible with qemu-kvm VAPIC */ + .version_id = 1, + .minimum_version_id = 1, + .minimum_version_id_old = 1, + .post_load = vapic_post_load, + .fields = (VMStateField[]) { + VMSTATE_STRUCT(rom_state, VAPICROMState, 0, vmstate_guest_rom, + GuestROMState), + VMSTATE_UINT32(state, VAPICROMState), + VMSTATE_UINT32(real_tpr_addr, VAPICROMState), + VMSTATE_UINT32(rom_state_vaddr, VAPICROMState), + VMSTATE_UINT32(vapic_paddr, VAPICROMState), + VMSTATE_UINT32(rom_state_paddr, VAPICROMState), + VMSTATE_END_OF_LIST() + } +}; + +static void vapic_class_init(ObjectClass *klass, void *data) +{ + SysBusDeviceClass *sc = SYS_BUS_DEVICE_CLASS(klass); + DeviceClass *dc = DEVICE_CLASS(klass); + + dc->no_user = 1; + dc->reset = vapic_reset; + dc->vmsd = &vmstate_vapic; + sc->init = vapic_init; +} + +static TypeInfo vapic_type = { + .name = "kvmvapic", + .parent = TYPE_SYS_BUS_DEVICE, + .instance_size = sizeof(VAPICROMState), + .class_init = vapic_class_init, +}; + +static void vapic_register(void) +{ + type_register_static(&vapic_type); +} + +type_init(vapic_register); diff --git a/hw/mc146818rtc.c b/hw/mc146818rtc.c index a46fdfc487..8b5cf8c81f 100644 --- a/hw/mc146818rtc.c +++ b/hw/mc146818rtc.c @@ -25,10 +25,13 @@ #include "qemu-timer.h" #include "sysemu.h" #include "pc.h" -#include "apic.h" #include "isa.h" #include "mc146818rtc.h" +#ifdef TARGET_I386 +#include "apic.h" +#endif + //#define DEBUG_CMOS //#define DEBUG_COALESCED diff --git a/hw/vhost.c b/hw/vhost.c index 5ece659f4a..8d3ba5b608 100644 --- a/hw/vhost.c +++ b/hw/vhost.c @@ -436,6 +436,14 @@ static bool vhost_section(MemoryRegionSection *section) && memory_region_is_ram(section->mr); } +static void vhost_begin(MemoryListener *listener) +{ +} + +static void vhost_commit(MemoryListener *listener) +{ +} + static void vhost_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -476,6 +484,11 @@ static void vhost_region_del(MemoryListener *listener, } } +static void vhost_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static int vhost_virtqueue_set_addr(struct vhost_dev *dev, struct vhost_virtqueue *vq, unsigned idx, bool enable_log) @@ -720,6 +733,18 @@ static void vhost_virtqueue_cleanup(struct vhost_dev *dev, 0, virtio_queue_get_desc_size(vdev, idx)); } +static void vhost_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void vhost_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) { uint64_t features; @@ -744,13 +769,19 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) hdev->features = features; hdev->memory_listener = (MemoryListener) { + .begin = vhost_begin, + .commit = vhost_commit, .region_add = vhost_region_add, .region_del = vhost_region_del, + .region_nop = vhost_region_nop, .log_start = vhost_log_start, .log_stop = vhost_log_stop, .log_sync = vhost_log_sync, .log_global_start = vhost_log_global_start, .log_global_stop = vhost_log_global_stop, + .eventfd_add = vhost_eventfd_add, + .eventfd_del = vhost_eventfd_del, + .priority = 10 }; hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions)); hdev->n_mem_sections = 0; @@ -759,7 +790,7 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force) hdev->log_size = 0; hdev->log_enabled = false; hdev->started = false; - memory_listener_register(&hdev->memory_listener); + memory_listener_register(&hdev->memory_listener, NULL); hdev->force = force; return 0; fail: @@ -328,6 +328,7 @@ void portio_list_init(PortioList *piolist, piolist->ports = callbacks; piolist->nr = 0; piolist->regions = g_new0(MemoryRegion *, n); + piolist->aliases = g_new0(MemoryRegion *, n); piolist->address_space = NULL; piolist->opaque = opaque; piolist->name = name; @@ -336,6 +337,7 @@ void portio_list_init(PortioList *piolist, void portio_list_destroy(PortioList *piolist) { g_free(piolist->regions); + g_free(piolist->aliases); } static void portio_list_add_1(PortioList *piolist, @@ -345,7 +347,7 @@ static void portio_list_add_1(PortioList *piolist, { MemoryRegionPortio *pio; MemoryRegionOps *ops; - MemoryRegion *region; + MemoryRegion *region, *alias; unsigned i; /* Copy the sub-list and null-terminate it. */ @@ -362,12 +364,20 @@ static void portio_list_add_1(PortioList *piolist, ops->old_portio = pio; region = g_new(MemoryRegion, 1); + alias = g_new(MemoryRegion, 1); + /* + * Use an alias so that the callback is called with an absolute address, + * rather than an offset relative to to start + off_low. + */ memory_region_init_io(region, ops, piolist->opaque, piolist->name, - off_high - off_low); - memory_region_set_offset(region, start + off_low); + UINT64_MAX); + memory_region_init_alias(alias, piolist->name, + region, start + off_low, off_high - off_low); memory_region_add_subregion(piolist->address_space, - start + off_low, region); - piolist->regions[piolist->nr++] = region; + start + off_low, alias); + piolist->regions[piolist->nr] = region; + piolist->aliases[piolist->nr] = alias; + ++piolist->nr; } void portio_list_add(PortioList *piolist, @@ -409,15 +419,19 @@ void portio_list_add(PortioList *piolist, void portio_list_del(PortioList *piolist) { - MemoryRegion *mr; + MemoryRegion *mr, *alias; unsigned i; for (i = 0; i < piolist->nr; ++i) { mr = piolist->regions[i]; - memory_region_del_subregion(piolist->address_space, mr); + alias = piolist->aliases[i]; + memory_region_del_subregion(piolist->address_space, alias); + memory_region_destroy(alias); memory_region_destroy(mr); g_free((MemoryRegionOps *)mr->ops); g_free(mr); + g_free(alias); piolist->regions[i] = NULL; + piolist->aliases[i] = NULL; } } @@ -60,6 +60,7 @@ typedef struct PortioList { struct MemoryRegion *address_space; unsigned nr; struct MemoryRegion **regions; + struct MemoryRegion **aliases; void *opaque; const char *name; } PortioList; @@ -28,6 +28,7 @@ #include "kvm.h" #include "bswap.h" #include "memory.h" +#include "exec-memory.h" /* This check must be after config-host.h is included */ #ifdef CONFIG_EVENTFD @@ -541,17 +542,26 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add) target_phys_addr_t start_addr = section->offset_within_address_space; ram_addr_t size = section->size; void *ram = NULL; + unsigned delta; /* kvm works in page size chunks, but the function may be called with sub-page size and unaligned start address. */ - size = TARGET_PAGE_ALIGN(size); - start_addr = TARGET_PAGE_ALIGN(start_addr); + delta = TARGET_PAGE_ALIGN(size) - size; + if (delta > size) { + return; + } + start_addr += delta; + size -= delta; + size &= TARGET_PAGE_MASK; + if (!size || (start_addr & ~TARGET_PAGE_MASK)) { + return; + } if (!memory_region_is_ram(mr)) { return; } - ram = memory_region_get_ram_ptr(mr) + section->offset_within_region; + ram = memory_region_get_ram_ptr(mr) + section->offset_within_region + delta; while (1) { mem = kvm_lookup_overlapping_slot(s, start_addr, start_addr + size); @@ -674,6 +684,14 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add) } } +static void kvm_begin(MemoryListener *listener) +{ +} + +static void kvm_commit(MemoryListener *listener) +{ +} + static void kvm_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -686,6 +704,11 @@ static void kvm_region_del(MemoryListener *listener, kvm_set_phys_mem(section, false); } +static void kvm_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static void kvm_log_sync(MemoryListener *listener, MemoryRegionSection *section) { @@ -713,14 +736,95 @@ static void kvm_log_global_stop(struct MemoryListener *listener) assert(r >= 0); } +static void kvm_mem_ioeventfd_add(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + assert(match_data && section->size == 4); + + r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space, + data, true); + if (r < 0) { + abort(); + } +} + +static void kvm_mem_ioeventfd_del(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space, + data, false); + if (r < 0) { + abort(); + } +} + +static void kvm_io_ioeventfd_add(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + int r; + + assert(match_data && section->size == 2); + + r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space, + data, true); + if (r < 0) { + abort(); + } +} + +static void kvm_io_ioeventfd_del(MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) + +{ + int r; + + r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space, + data, false); + if (r < 0) { + abort(); + } +} + +static void kvm_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + if (section->address_space == get_system_memory()) { + kvm_mem_ioeventfd_add(section, match_data, data, fd); + } else { + kvm_io_ioeventfd_add(section, match_data, data, fd); + } +} + +static void kvm_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ + if (section->address_space == get_system_memory()) { + kvm_mem_ioeventfd_del(section, match_data, data, fd); + } else { + kvm_io_ioeventfd_del(section, match_data, data, fd); + } +} + static MemoryListener kvm_memory_listener = { + .begin = kvm_begin, + .commit = kvm_commit, .region_add = kvm_region_add, .region_del = kvm_region_del, + .region_nop = kvm_region_nop, .log_start = kvm_log_start, .log_stop = kvm_log_stop, .log_sync = kvm_log_sync, .log_global_start = kvm_log_global_start, .log_global_stop = kvm_log_global_stop, + .eventfd_add = kvm_eventfd_add, + .eventfd_del = kvm_eventfd_del, + .priority = 10, }; static void kvm_handle_interrupt(CPUState *env, int mask) @@ -965,7 +1069,7 @@ int kvm_init(void) } kvm_state = s; - memory_listener_register(&kvm_memory_listener); + memory_listener_register(&kvm_memory_listener, NULL); s->many_ioeventfds = kvm_check_many_ioeventfds(); @@ -1118,8 +1222,6 @@ int kvm_cpu_exec(CPUState *env) return EXCP_HLT; } - cpu_single_env = env; - do { if (env->kvm_vcpu_dirty) { kvm_arch_put_registers(env, KVM_PUT_RUNTIME_STATE); @@ -1136,13 +1238,11 @@ int kvm_cpu_exec(CPUState *env) */ qemu_cpu_kick_self(); } - cpu_single_env = NULL; qemu_mutex_unlock_iothread(); run_ret = kvm_vcpu_ioctl(env, KVM_RUN, 0); qemu_mutex_lock_iothread(); - cpu_single_env = env; kvm_arch_post_run(env, run); kvm_flush_coalesced_mmio_buffer(); @@ -1206,7 +1306,6 @@ int kvm_cpu_exec(CPUState *env) } env->exit_request = 0; - cpu_single_env = NULL; return ret; } @@ -27,8 +27,8 @@ unsigned memory_region_transaction_depth = 0; static bool memory_region_update_pending = false; static bool global_dirty_log = false; -static QLIST_HEAD(, MemoryListener) memory_listeners - = QLIST_HEAD_INITIALIZER(memory_listeners); +static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners + = QTAILQ_HEAD_INITIALIZER(memory_listeners); typedef struct AddrRange AddrRange; @@ -82,6 +82,71 @@ static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2) return addrrange_make(start, int128_sub(end, start)); } +enum ListenerDirection { Forward, Reverse }; + +static bool memory_listener_match(MemoryListener *listener, + MemoryRegionSection *section) +{ + return !listener->address_space_filter + || listener->address_space_filter == section->address_space; +} + +#define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + _listener->_callback(_listener, ##_args); \ + } \ + break; \ + default: \ + abort(); \ + } \ + } while (0) + +#define MEMORY_LISTENER_CALL(_callback, _direction, _section, _args...) \ + do { \ + MemoryListener *_listener; \ + \ + switch (_direction) { \ + case Forward: \ + QTAILQ_FOREACH(_listener, &memory_listeners, link) { \ + if (memory_listener_match(_listener, _section)) { \ + _listener->_callback(_listener, _section, ##_args); \ + } \ + } \ + break; \ + case Reverse: \ + QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \ + memory_listeners, link) { \ + if (memory_listener_match(_listener, _section)) { \ + _listener->_callback(_listener, _section, ##_args); \ + } \ + } \ + break; \ + default: \ + abort(); \ + } \ + } while (0) + +#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \ + MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) { \ + .mr = (fr)->mr, \ + .address_space = (as)->root, \ + .offset_within_region = (fr)->offset_in_region, \ + .size = int128_get64((fr)->addr.size), \ + .offset_within_address_space = int128_get64((fr)->addr.start), \ + .readonly = (fr)->readonly, \ + })) + struct CoalescedMemoryRange { AddrRange addr; QTAILQ_ENTRY(CoalescedMemoryRange) link; @@ -158,22 +223,12 @@ typedef struct AddressSpaceOps AddressSpaceOps; /* A system address space - I/O, memory, etc. */ struct AddressSpace { - const AddressSpaceOps *ops; MemoryRegion *root; FlatView current_map; int ioeventfd_nb; MemoryRegionIoeventfd *ioeventfds; }; -struct AddressSpaceOps { - void (*range_add)(AddressSpace *as, FlatRange *fr); - void (*range_del)(AddressSpace *as, FlatRange *fr); - void (*log_start)(AddressSpace *as, FlatRange *fr); - void (*log_stop)(AddressSpace *as, FlatRange *fr); - void (*ioeventfd_add)(AddressSpace *as, MemoryRegionIoeventfd *fd); - void (*ioeventfd_del)(AddressSpace *as, MemoryRegionIoeventfd *fd); -}; - #define FOR_EACH_FLAT_RANGE(var, view) \ for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var) @@ -305,74 +360,7 @@ static void access_with_adjusted_size(target_phys_addr_t addr, } } -static void as_memory_range_add(AddressSpace *as, FlatRange *fr) -{ - MemoryRegionSection section = { - .mr = fr->mr, - .offset_within_address_space = int128_get64(fr->addr.start), - .offset_within_region = fr->offset_in_region, - .size = int128_get64(fr->addr.size), - }; - - cpu_register_physical_memory_log(§ion, fr->readable, fr->readonly); -} - -static void as_memory_range_del(AddressSpace *as, FlatRange *fr) -{ - MemoryRegionSection section = { - .mr = &io_mem_unassigned, - .offset_within_address_space = int128_get64(fr->addr.start), - .offset_within_region = int128_get64(fr->addr.start), - .size = int128_get64(fr->addr.size), - }; - - cpu_register_physical_memory_log(§ion, true, false); -} - -static void as_memory_log_start(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_log_stop(AddressSpace *as, FlatRange *fr) -{ -} - -static void as_memory_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - assert(fd->match_data && int128_get64(fd->addr.size) == 4); - - r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start), - fd->data, true); - if (r < 0) { - abort(); - } -} - -static void as_memory_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start), - fd->data, false); - if (r < 0) { - abort(); - } -} - -static const AddressSpaceOps address_space_ops_memory = { - .range_add = as_memory_range_add, - .range_del = as_memory_range_del, - .log_start = as_memory_log_start, - .log_stop = as_memory_log_stop, - .ioeventfd_add = as_memory_ioeventfd_add, - .ioeventfd_del = as_memory_ioeventfd_del, -}; - -static AddressSpace address_space_memory = { - .ops = &address_space_ops_memory, -}; +static AddressSpace address_space_memory; static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset, unsigned width, bool write) @@ -401,17 +389,17 @@ static void memory_region_iorange_read(IORange *iorange, *data = ((uint64_t)1 << (width * 8)) - 1; if (mrp) { - *data = mrp->read(mr->opaque, offset + mr->offset); + *data = mrp->read(mr->opaque, offset); } else if (width == 2) { mrp = find_portio(mr, offset, 1, false); assert(mrp); - *data = mrp->read(mr->opaque, offset + mr->offset) | - (mrp->read(mr->opaque, offset + mr->offset + 1) << 8); + *data = mrp->read(mr->opaque, offset) | + (mrp->read(mr->opaque, offset + 1) << 8); } return; } *data = 0; - access_with_adjusted_size(offset + mr->offset, data, width, + access_with_adjusted_size(offset, data, width, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_accessor, mr); @@ -428,73 +416,27 @@ static void memory_region_iorange_write(IORange *iorange, const MemoryRegionPortio *mrp = find_portio(mr, offset, width, true); if (mrp) { - mrp->write(mr->opaque, offset + mr->offset, data); + mrp->write(mr->opaque, offset, data); } else if (width == 2) { mrp = find_portio(mr, offset, 1, false); assert(mrp); - mrp->write(mr->opaque, offset + mr->offset, data & 0xff); - mrp->write(mr->opaque, offset + mr->offset + 1, data >> 8); + mrp->write(mr->opaque, offset, data & 0xff); + mrp->write(mr->opaque, offset + 1, data >> 8); } return; } - access_with_adjusted_size(offset + mr->offset, &data, width, + access_with_adjusted_size(offset, &data, width, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_accessor, mr); } -static const IORangeOps memory_region_iorange_ops = { +const IORangeOps memory_region_iorange_ops = { .read = memory_region_iorange_read, .write = memory_region_iorange_write, }; -static void as_io_range_add(AddressSpace *as, FlatRange *fr) -{ - iorange_init(&fr->mr->iorange, &memory_region_iorange_ops, - int128_get64(fr->addr.start), int128_get64(fr->addr.size)); - ioport_register(&fr->mr->iorange); -} - -static void as_io_range_del(AddressSpace *as, FlatRange *fr) -{ - isa_unassign_ioport(int128_get64(fr->addr.start), - int128_get64(fr->addr.size)); -} - -static void as_io_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - assert(fd->match_data && int128_get64(fd->addr.size) == 2); - - r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start), - fd->data, true); - if (r < 0) { - abort(); - } -} - -static void as_io_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd) -{ - int r; - - r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start), - fd->data, false); - if (r < 0) { - abort(); - } -} - -static const AddressSpaceOps address_space_ops_io = { - .range_add = as_io_range_add, - .range_del = as_io_range_del, - .ioeventfd_add = as_io_ioeventfd_add, - .ioeventfd_del = as_io_ioeventfd_del, -}; - -static AddressSpace address_space_io = { - .ops = &address_space_ops_io, -}; +static AddressSpace address_space_io; static AddressSpace *memory_region_to_address_space(MemoryRegion *mr) { @@ -621,6 +563,8 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, unsigned fds_old_nb) { unsigned iold, inew; + MemoryRegionIoeventfd *fd; + MemoryRegionSection section; /* Generate a symmetric difference of the old and new fd sets, adding * and deleting as necessary. @@ -632,13 +576,27 @@ static void address_space_add_del_ioeventfds(AddressSpace *as, && (inew == fds_new_nb || memory_region_ioeventfd_before(fds_old[iold], fds_new[inew]))) { - as->ops->ioeventfd_del(as, &fds_old[iold]); + fd = &fds_old[iold]; + section = (MemoryRegionSection) { + .address_space = as->root, + .offset_within_address_space = int128_get64(fd->addr.start), + .size = int128_get64(fd->addr.size), + }; + MEMORY_LISTENER_CALL(eventfd_del, Forward, §ion, + fd->match_data, fd->data, fd->fd); ++iold; } else if (inew < fds_new_nb && (iold == fds_old_nb || memory_region_ioeventfd_before(fds_new[inew], fds_old[iold]))) { - as->ops->ioeventfd_add(as, &fds_new[inew]); + fd = &fds_new[inew]; + section = (MemoryRegionSection) { + .address_space = as->root, + .offset_within_address_space = int128_get64(fd->addr.start), + .size = int128_get64(fd->addr.size), + }; + MEMORY_LISTENER_CALL(eventfd_add, Reverse, §ion, + fd->match_data, fd->data, fd->fd); ++inew; } else { ++iold; @@ -678,32 +636,6 @@ static void address_space_update_ioeventfds(AddressSpace *as) as->ioeventfd_nb = ioeventfd_nb; } -typedef void ListenerCallback(MemoryListener *listener, - MemoryRegionSection *mrs); - -/* Want "void (&MemoryListener::*callback)(const MemoryRegionSection& s)" */ -static void memory_listener_update_region(FlatRange *fr, AddressSpace *as, - size_t callback_offset) -{ - MemoryRegionSection section = { - .mr = fr->mr, - .address_space = as->root, - .offset_within_region = fr->offset_in_region, - .size = int128_get64(fr->addr.size), - .offset_within_address_space = int128_get64(fr->addr.start), - }; - MemoryListener *listener; - - QLIST_FOREACH(listener, &memory_listeners, link) { - ListenerCallback *callback - = *(ListenerCallback **)((void *)listener + callback_offset); - callback(listener, §ion); - } -} - -#define MEMORY_LISTENER_UPDATE_REGION(fr, as, callback) \ - memory_listener_update_region(fr, as, offsetof(MemoryListener, callback)) - static void address_space_update_topology_pass(AddressSpace *as, FlatView old_view, FlatView new_view, @@ -736,8 +668,7 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In old, but (not in new, or in new but attributes changed). */ if (!adding) { - MEMORY_LISTENER_UPDATE_REGION(frold, as, region_del); - as->ops->range_del(as, frold); + MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del); } ++iold; @@ -745,12 +676,11 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In both (logging may have changed) */ if (adding) { + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop); if (frold->dirty_log_mask && !frnew->dirty_log_mask) { - MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_stop); - as->ops->log_stop(as, frnew); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop); } else if (frnew->dirty_log_mask && !frold->dirty_log_mask) { - as->ops->log_start(as, frnew); - MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_start); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start); } } @@ -760,8 +690,7 @@ static void address_space_update_topology_pass(AddressSpace *as, /* In new */ if (adding) { - as->ops->range_add(as, frnew); - MEMORY_LISTENER_UPDATE_REGION(frnew, as, region_add); + MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add); } ++inew; @@ -794,6 +723,8 @@ static void memory_region_update_topology(MemoryRegion *mr) return; } + MEMORY_LISTENER_CALL_GLOBAL(begin, Forward); + if (address_space_memory.root) { address_space_update_topology(&address_space_memory); } @@ -801,6 +732,8 @@ static void memory_region_update_topology(MemoryRegion *mr) address_space_update_topology(&address_space_io); } + MEMORY_LISTENER_CALL_GLOBAL(commit, Forward); + memory_region_update_pending = false; } @@ -863,7 +796,6 @@ void memory_region_init(MemoryRegion *mr, mr->size = int128_2_64(); } mr->addr = 0; - mr->offset = 0; mr->subpage = false; mr->enabled = true; mr->terminates = false; @@ -925,7 +857,7 @@ static uint64_t memory_region_dispatch_read1(MemoryRegion *mr, } /* FIXME: support unaligned access */ - access_with_adjusted_size(addr + mr->offset, &data, size, + access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_read_accessor, mr); @@ -979,7 +911,7 @@ static void memory_region_dispatch_write(MemoryRegion *mr, } /* FIXME: support unaligned access */ - access_with_adjusted_size(addr + mr->offset, &data, size, + access_with_adjusted_size(addr, &data, size, mr->ops->impl.min_access_size, mr->ops->impl.max_access_size, memory_region_write_accessor, mr); @@ -1122,11 +1054,6 @@ bool memory_region_is_rom(MemoryRegion *mr) return mr->ram && mr->readonly; } -void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset) -{ - mr->offset = offset; -} - void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client) { uint8_t mask = 1 << client; @@ -1156,7 +1083,8 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr) FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) { if (fr->mr == mr) { - MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, log_sync); + MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, + Forward, log_sync); } } } @@ -1474,6 +1402,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space, fr->addr.start)); ret.size = int128_get64(range.size); ret.offset_within_address_space = int128_get64(range.start); + ret.readonly = fr->readonly; return ret; } @@ -1483,30 +1412,20 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space) FlatRange *fr; FOR_EACH_FLAT_RANGE(fr, &as->current_map) { - MEMORY_LISTENER_UPDATE_REGION(fr, as, log_sync); + MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync); } } void memory_global_dirty_log_start(void) { - MemoryListener *listener; - - cpu_physical_memory_set_dirty_tracking(1); global_dirty_log = true; - QLIST_FOREACH(listener, &memory_listeners, link) { - listener->log_global_start(listener); - } + MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward); } void memory_global_dirty_log_stop(void) { - MemoryListener *listener; - global_dirty_log = false; - QLIST_FOREACH(listener, &memory_listeners, link) { - listener->log_global_stop(listener); - } - cpu_physical_memory_set_dirty_tracking(0); + MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse); } static void listener_add_address_space(MemoryListener *listener, @@ -1524,21 +1443,36 @@ static void listener_add_address_space(MemoryListener *listener, .offset_within_region = fr->offset_in_region, .size = int128_get64(fr->addr.size), .offset_within_address_space = int128_get64(fr->addr.start), + .readonly = fr->readonly, }; listener->region_add(listener, §ion); } } -void memory_listener_register(MemoryListener *listener) +void memory_listener_register(MemoryListener *listener, MemoryRegion *filter) { - QLIST_INSERT_HEAD(&memory_listeners, listener, link); + MemoryListener *other = NULL; + + listener->address_space_filter = filter; + if (QTAILQ_EMPTY(&memory_listeners) + || listener->priority >= QTAILQ_LAST(&memory_listeners, + memory_listeners)->priority) { + QTAILQ_INSERT_TAIL(&memory_listeners, listener, link); + } else { + QTAILQ_FOREACH(other, &memory_listeners, link) { + if (listener->priority < other->priority) { + break; + } + } + QTAILQ_INSERT_BEFORE(other, listener, link); + } listener_add_address_space(listener, &address_space_memory); listener_add_address_space(listener, &address_space_io); } void memory_listener_unregister(MemoryListener *listener) { - QLIST_REMOVE(listener, link); + QTAILQ_REMOVE(&memory_listeners, listener, link); } void set_system_memory_map(MemoryRegion *mr) @@ -115,7 +115,6 @@ struct MemoryRegion { MemoryRegion *parent; Int128 size; target_phys_addr_t addr; - target_phys_addr_t offset; void (*destructor)(MemoryRegion *mr); ram_addr_t ram_addr; IORange iorange; @@ -161,6 +160,7 @@ typedef struct MemoryRegionSection MemoryRegionSection; * @size: the size of the section; will not exceed @mr's boundaries * @offset_within_address_space: the address of the first byte of the section * relative to the region's address space + * @readonly: writes to this section are ignored */ struct MemoryRegionSection { MemoryRegion *mr; @@ -168,6 +168,7 @@ struct MemoryRegionSection { target_phys_addr_t offset_within_region; uint64_t size; target_phys_addr_t offset_within_address_space; + bool readonly; }; typedef struct MemoryListener MemoryListener; @@ -179,14 +180,24 @@ typedef struct MemoryListener MemoryListener; * Use with memory_listener_register() and memory_listener_unregister(). */ struct MemoryListener { + void (*begin)(MemoryListener *listener); + void (*commit)(MemoryListener *listener); void (*region_add)(MemoryListener *listener, MemoryRegionSection *section); void (*region_del)(MemoryListener *listener, MemoryRegionSection *section); + void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section); void (*log_start)(MemoryListener *listener, MemoryRegionSection *section); void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section); void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section); void (*log_global_start)(MemoryListener *listener); void (*log_global_stop)(MemoryListener *listener); - QLIST_ENTRY(MemoryListener) link; + void (*eventfd_add)(MemoryListener *listener, MemoryRegionSection *section, + bool match_data, uint64_t data, int fd); + void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section, + bool match_data, uint64_t data, int fd); + /* Lower = earlier (during add), later (during del) */ + unsigned priority; + MemoryRegion *address_space_filter; + QTAILQ_ENTRY(MemoryListener) link; }; /** @@ -359,14 +370,6 @@ bool memory_region_is_rom(MemoryRegion *mr); void *memory_region_get_ram_ptr(MemoryRegion *mr); /** - * memory_region_set_offset: Sets an offset to be added to MemoryRegionOps - * callbacks. - * - * This function is deprecated and should not be used in new code. - */ -void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset); - -/** * memory_region_set_log: Turn dirty logging on or off for a region. * * Turns dirty logging on or off for a specified client (display, migration). @@ -686,8 +689,9 @@ void memory_region_transaction_commit(void); * space * * @listener: an object containing the callbacks to be called + * @filter: if non-%NULL, only regions in this address space will be observed */ -void memory_listener_register(MemoryListener *listener); +void memory_listener_register(MemoryListener *listener, MemoryRegion *filter); /** * memory_listener_unregister: undo the effect of memory_listener_register() @@ -1949,7 +1949,7 @@ static void tlb_info(Monitor *mon) #endif -#if defined(TARGET_SPARC) || defined(TARGET_PPC) +#if defined(TARGET_SPARC) || defined(TARGET_PPC) || defined(TARGET_XTENSA) static void tlb_info(Monitor *mon) { CPUState *env1 = mon_get_cpu(); @@ -2396,7 +2396,7 @@ static mon_cmd_t info_cmds[] = { .mhandler.info = hmp_info_pci, }, #if defined(TARGET_I386) || defined(TARGET_SH4) || defined(TARGET_SPARC) || \ - defined(TARGET_PPC) + defined(TARGET_PPC) || defined(TARGET_XTENSA) { .name = "tlb", .args_type = "", diff --git a/pc-bios/kvmvapic.bin b/pc-bios/kvmvapic.bin Binary files differnew file mode 100755 index 0000000000..045f5c2884 --- /dev/null +++ b/pc-bios/kvmvapic.bin diff --git a/pc-bios/optionrom/Makefile b/pc-bios/optionrom/Makefile index 2caf7e6b69..f6b402713b 100644 --- a/pc-bios/optionrom/Makefile +++ b/pc-bios/optionrom/Makefile @@ -14,7 +14,7 @@ CFLAGS += -I$(SRC_PATH) CFLAGS += $(call cc-option, $(CFLAGS), -fno-stack-protector) QEMU_CFLAGS = $(CFLAGS) -build-all: multiboot.bin linuxboot.bin +build-all: multiboot.bin linuxboot.bin kvmvapic.bin # suppress auto-removal of intermediate files .SECONDARY: diff --git a/pc-bios/optionrom/kvmvapic.S b/pc-bios/optionrom/kvmvapic.S new file mode 100644 index 0000000000..aa17a402df --- /dev/null +++ b/pc-bios/optionrom/kvmvapic.S @@ -0,0 +1,335 @@ +# +# Local APIC acceleration for Windows XP and related guests +# +# Copyright 2011 Red Hat, Inc. and/or its affiliates +# +# Author: Avi Kivity <avi@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2, or (at your +# option) any later version. See the COPYING file in the top-level directory. +# + +#include "optionrom.h" + +OPTION_ROM_START + + # clear vapic area: firmware load using rep insb may cause + # stale tpr/isr/irr data to corrupt the vapic area. + push %es + push %cs + pop %es + xor %ax, %ax + mov $vapic_size/2, %cx + lea vapic, %di + cld + rep stosw + pop %es + + # announce presence to the hypervisor + mov $vapic_base, %ax + out %ax, $0x7e + + lret + + .code32 +vapic_size = 2*4096 + +.macro fixup delta=-4 +777: + .text 1 + .long 777b + \delta - vapic_base + .text 0 +.endm + +.macro reenable_vtpr + out %al, $0x7e +.endm + +.text 1 + fixup_start = . +.text 0 + +.align 16 + +vapic_base: + .ascii "kvm aPiC" + + /* relocation data */ + .long vapic_base ; fixup + .long fixup_start ; fixup + .long fixup_end ; fixup + + .long vapic ; fixup + .long vapic_size +vcpu_shift: + .long 0 +real_tpr: + .long 0 + .long up_set_tpr ; fixup + .long up_set_tpr_eax ; fixup + .long up_get_tpr_eax ; fixup + .long up_get_tpr_ecx ; fixup + .long up_get_tpr_edx ; fixup + .long up_get_tpr_ebx ; fixup + .long 0 /* esp. won't work. */ + .long up_get_tpr_ebp ; fixup + .long up_get_tpr_esi ; fixup + .long up_get_tpr_edi ; fixup + .long up_get_tpr_stack ; fixup + .long mp_set_tpr ; fixup + .long mp_set_tpr_eax ; fixup + .long mp_get_tpr_eax ; fixup + .long mp_get_tpr_ecx ; fixup + .long mp_get_tpr_edx ; fixup + .long mp_get_tpr_ebx ; fixup + .long 0 /* esp. won't work. */ + .long mp_get_tpr_ebp ; fixup + .long mp_get_tpr_esi ; fixup + .long mp_get_tpr_edi ; fixup + .long mp_get_tpr_stack ; fixup + +.macro kvm_hypercall + .byte 0x0f, 0x01, 0xc1 +.endm + +kvm_hypercall_vapic_poll_irq = 1 + +pcr_cpu = 0x51 + +.align 64 + +mp_get_tpr_eax: + pushf + cli + reenable_vtpr + push %ecx + + fs/movzbl pcr_cpu, %eax + + mov vcpu_shift, %ecx ; fixup + shl %cl, %eax + testb $1, vapic+4(%eax) ; fixup delta=-5 + jz mp_get_tpr_bad + movzbl vapic(%eax), %eax ; fixup + +mp_get_tpr_out: + pop %ecx + popf + ret + +mp_get_tpr_bad: + mov real_tpr, %eax ; fixup + mov (%eax), %eax + jmp mp_get_tpr_out + +mp_get_tpr_ebx: + mov %eax, %ebx + call mp_get_tpr_eax + xchg %eax, %ebx + ret + +mp_get_tpr_ecx: + mov %eax, %ecx + call mp_get_tpr_eax + xchg %eax, %ecx + ret + +mp_get_tpr_edx: + mov %eax, %edx + call mp_get_tpr_eax + xchg %eax, %edx + ret + +mp_get_tpr_esi: + mov %eax, %esi + call mp_get_tpr_eax + xchg %eax, %esi + ret + +mp_get_tpr_edi: + mov %eax, %edi + call mp_get_tpr_edi + xchg %eax, %edi + ret + +mp_get_tpr_ebp: + mov %eax, %ebp + call mp_get_tpr_eax + xchg %eax, %ebp + ret + +mp_get_tpr_stack: + call mp_get_tpr_eax + xchg %eax, 4(%esp) + ret + +mp_set_tpr_eax: + push %eax + call mp_set_tpr + ret + +mp_set_tpr: + pushf + push %eax + push %ecx + push %edx + push %ebx + cli + reenable_vtpr + +mp_set_tpr_failed: + fs/movzbl pcr_cpu, %edx + + mov vcpu_shift, %ecx ; fixup + shl %cl, %edx + + testb $1, vapic+4(%edx) ; fixup delta=-5 + jz mp_set_tpr_bad + + mov vapic(%edx), %eax ; fixup + + mov %eax, %ebx + mov 24(%esp), %bl + + /* %ebx = new vapic (%bl = tpr, %bh = isr, %b3 = irr) */ + + lock cmpxchg %ebx, vapic(%edx) ; fixup + jnz mp_set_tpr_failed + + /* compute ppr */ + cmp %bh, %bl + jae mp_tpr_is_bigger +mp_isr_is_bigger: + mov %bh, %bl +mp_tpr_is_bigger: + /* %bl = ppr */ + rol $8, %ebx + /* now: %bl = irr, %bh = ppr */ + cmp %bh, %bl + ja mp_set_tpr_poll_irq + +mp_set_tpr_out: + pop %ebx + pop %edx + pop %ecx + pop %eax + popf + ret $4 + +mp_set_tpr_poll_irq: + mov $kvm_hypercall_vapic_poll_irq, %eax + kvm_hypercall + jmp mp_set_tpr_out + +mp_set_tpr_bad: + mov 24(%esp), %ecx + mov real_tpr, %eax ; fixup + mov %ecx, (%eax) + jmp mp_set_tpr_out + +up_get_tpr_eax: + reenable_vtpr + movzbl vapic, %eax ; fixup + ret + +up_get_tpr_ebx: + reenable_vtpr + movzbl vapic, %ebx ; fixup + ret + +up_get_tpr_ecx: + reenable_vtpr + movzbl vapic, %ecx ; fixup + ret + +up_get_tpr_edx: + reenable_vtpr + movzbl vapic, %edx ; fixup + ret + +up_get_tpr_esi: + reenable_vtpr + movzbl vapic, %esi ; fixup + ret + +up_get_tpr_edi: + reenable_vtpr + movzbl vapic, %edi ; fixup + ret + +up_get_tpr_ebp: + reenable_vtpr + movzbl vapic, %ebp ; fixup + ret + +up_get_tpr_stack: + reenable_vtpr + movzbl vapic, %eax ; fixup + xchg %eax, 4(%esp) + ret + +up_set_tpr_eax: + push %eax + call up_set_tpr + ret + +up_set_tpr: + pushf + push %eax + push %ebx + reenable_vtpr + +up_set_tpr_failed: + mov vapic, %eax ; fixup + + mov %eax, %ebx + mov 16(%esp), %bl + + /* %ebx = new vapic (%bl = tpr, %bh = isr, %b3 = irr) */ + + lock cmpxchg %ebx, vapic ; fixup + jnz up_set_tpr_failed + + /* compute ppr */ + cmp %bh, %bl + jae up_tpr_is_bigger +up_isr_is_bigger: + mov %bh, %bl +up_tpr_is_bigger: + /* %bl = ppr */ + rol $8, %ebx + /* now: %bl = irr, %bh = ppr */ + cmp %bh, %bl + ja up_set_tpr_poll_irq + +up_set_tpr_out: + pop %ebx + pop %eax + popf + ret $4 + +up_set_tpr_poll_irq: + mov $kvm_hypercall_vapic_poll_irq, %eax + kvm_hypercall + jmp up_set_tpr_out + +.text 1 + fixup_end = . +.text 0 + +/* + * vapic format: + * per-vcpu records of size 2^vcpu shift. + * byte 0: tpr (r/w) + * byte 1: highest in-service interrupt (isr) (r/o); bits 3:0 are zero + * byte 2: zero (r/o) + * byte 3: highest pending interrupt (irr) (r/o) + */ +.text 2 + +.align 128 + +vapic: +. = . + vapic_size + +OPTION_ROM_END diff --git a/pc-bios/optionrom/optionrom.h b/pc-bios/optionrom/optionrom.h index aa783deed1..3daf7da495 100644 --- a/pc-bios/optionrom/optionrom.h +++ b/pc-bios/optionrom/optionrom.h @@ -124,7 +124,8 @@ movw %ax, %ds; #define OPTION_ROM_END \ - .align 512, 0; \ + .byte 0; \ + .align 512, 0; \ _end: #define BOOT_ROM_END \ diff --git a/target-i386/cpu.h b/target-i386/cpu.h index 37dde79581..196b0c5c40 100644 --- a/target-i386/cpu.h +++ b/target-i386/cpu.h @@ -482,6 +482,7 @@ #define CPU_INTERRUPT_VIRQ CPU_INTERRUPT_TGT_INT_0 #define CPU_INTERRUPT_INIT CPU_INTERRUPT_TGT_INT_1 #define CPU_INTERRUPT_SIPI CPU_INTERRUPT_TGT_INT_2 +#define CPU_INTERRUPT_TPR CPU_INTERRUPT_TGT_INT_3 enum { @@ -613,6 +614,11 @@ typedef struct { #define NB_MMU_MODES 2 +typedef enum TPRAccess { + TPR_ACCESS_READ, + TPR_ACCESS_WRITE, +} TPRAccess; + typedef struct CPUX86State { /* standard registers */ target_ulong regs[CPU_NB_REGS]; @@ -772,6 +778,8 @@ typedef struct CPUX86State { XMMReg ymmh_regs[CPU_NB_REGS]; uint64_t xcr0; + + TPRAccess tpr_access_type; } CPUX86State; CPUX86State *cpu_x86_init(const char *cpu_model); @@ -1064,4 +1072,6 @@ void svm_check_intercept(CPUState *env1, uint32_t type); uint32_t cpu_cc_compute_all(CPUState *env1, int op); +void cpu_report_tpr_access(CPUState *env, TPRAccess access); + #endif /* CPU_I386_H */ diff --git a/target-i386/helper.c b/target-i386/helper.c index 2586aff700..d12c9621bd 100644 --- a/target-i386/helper.c +++ b/target-i386/helper.c @@ -1189,6 +1189,22 @@ void cpu_x86_inject_mce(Monitor *mon, CPUState *cenv, int bank, } } } + +void cpu_report_tpr_access(CPUState *env, TPRAccess access) +{ + TranslationBlock *tb; + + if (kvm_enabled()) { + env->tpr_access_type = access; + + cpu_interrupt(env, CPU_INTERRUPT_TPR); + } else { + tb = tb_find_pc(env->mem_io_pc); + cpu_restore_state(tb, env, env->mem_io_pc); + + apic_handle_tpr_access_report(env->apic_state, env->eip, access); + } +} #endif /* !CONFIG_USER_ONLY */ static void mce_init(CPUX86State *cenv) diff --git a/target-i386/kvm.c b/target-i386/kvm.c index 981192ddf8..9a732078f3 100644 --- a/target-i386/kvm.c +++ b/target-i386/kvm.c @@ -1635,8 +1635,10 @@ void kvm_arch_pre_run(CPUState *env, struct kvm_run *run) } if (!kvm_irqchip_in_kernel()) { - /* Force the VCPU out of its inner loop to process the INIT request */ - if (env->interrupt_request & CPU_INTERRUPT_INIT) { + /* Force the VCPU out of its inner loop to process any INIT requests + * or pending TPR access reports. */ + if (env->interrupt_request & + (CPU_INTERRUPT_INIT | CPU_INTERRUPT_TPR)) { env->exit_request = 1; } @@ -1730,6 +1732,12 @@ int kvm_arch_process_async_events(CPUState *env) kvm_cpu_synchronize_state(env); do_cpu_sipi(env); } + if (env->interrupt_request & CPU_INTERRUPT_TPR) { + env->interrupt_request &= ~CPU_INTERRUPT_TPR; + kvm_cpu_synchronize_state(env); + apic_handle_tpr_access_report(env->apic_state, env->eip, + env->tpr_access_type); + } return env->halted; } @@ -1746,6 +1754,16 @@ static int kvm_handle_halt(CPUState *env) return 0; } +static int kvm_handle_tpr_access(CPUState *env) +{ + struct kvm_run *run = env->kvm_run; + + apic_handle_tpr_access_report(env->apic_state, run->tpr_access.rip, + run->tpr_access.is_write ? TPR_ACCESS_WRITE + : TPR_ACCESS_READ); + return 1; +} + int kvm_arch_insert_sw_breakpoint(CPUState *env, struct kvm_sw_breakpoint *bp) { static const uint8_t int3 = 0xcc; @@ -1950,6 +1968,9 @@ int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run) case KVM_EXIT_SET_TPR: ret = 0; break; + case KVM_EXIT_TPR_ACCESS: + ret = kvm_handle_tpr_access(env); + break; case KVM_EXIT_FAIL_ENTRY: code = run->fail_entry.hardware_entry_failure_reason; fprintf(stderr, "KVM: entry failed, hardware error 0x%" PRIx64 "\n", @@ -1987,6 +2008,7 @@ int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run) bool kvm_arch_stop_on_emulation_error(CPUState *env) { + kvm_cpu_synchronize_state(env); return !(env->cr[0] & CR0_PE_MASK) || ((env->segs[R_CS].selector & 3) != 3); } diff --git a/target-xtensa/core-dc232b.c b/target-xtensa/core-dc232b.c index 4d9bd559d4..b723c4ca7b 100644 --- a/target-xtensa/core-dc232b.c +++ b/target-xtensa/core-dc232b.c @@ -22,6 +22,7 @@ static const XtensaConfig dc232b = { EXCEPTIONS_SECTION, INTERRUPTS_SECTION, TLB_SECTION, + DEBUG_SECTION, .clock_freq_khz = 10000, }; diff --git a/target-xtensa/core-fsf.c b/target-xtensa/core-fsf.c index 7650462f2f..88de4dd4ba 100644 --- a/target-xtensa/core-fsf.c +++ b/target-xtensa/core-fsf.c @@ -16,6 +16,7 @@ static const XtensaConfig fsf = { EXCEPTIONS_SECTION, INTERRUPTS_SECTION, TLB_SECTION, + DEBUG_SECTION, .clock_freq_khz = 10000, }; diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h index 0db83a6fd7..fb8a727c66 100644 --- a/target-xtensa/cpu.h +++ b/target-xtensa/cpu.h @@ -126,6 +126,10 @@ enum { RASID = 90, ITLBCFG = 91, DTLBCFG = 92, + IBREAKENABLE = 96, + IBREAKA = 128, + DBREAKA = 144, + DBREAKC = 160, EPC1 = 177, DEPC = 192, EPS2 = 194, @@ -137,8 +141,11 @@ enum { PS = 230, VECBASE = 231, EXCCAUSE = 232, + DEBUGCAUSE = 233, CCOUNT = 234, PRID = 235, + ICOUNT = 236, + ICOUNTLEVEL = 237, EXCVADDR = 238, CCOMPARE = 240, }; @@ -161,12 +168,27 @@ enum { #define PS_WOE 0x40000 +#define DEBUGCAUSE_IC 0x1 +#define DEBUGCAUSE_IB 0x2 +#define DEBUGCAUSE_DB 0x4 +#define DEBUGCAUSE_BI 0x8 +#define DEBUGCAUSE_BN 0x10 +#define DEBUGCAUSE_DI 0x20 +#define DEBUGCAUSE_DBNUM 0xf00 +#define DEBUGCAUSE_DBNUM_SHIFT 8 + +#define DBREAKC_SB 0x80000000 +#define DBREAKC_LB 0x40000000 +#define DBREAKC_SB_LB (DBREAKC_SB | DBREAKC_LB) +#define DBREAKC_MASK 0x3f + #define MAX_NAREG 64 #define MAX_NINTERRUPT 32 #define MAX_NLEVEL 6 #define MAX_NNMI 1 #define MAX_NCCOMPARE 3 #define MAX_TLB_WAY_SIZE 8 +#define MAX_NDBREAK 2 #define REGION_PAGE_MASK 0xe0000000 @@ -186,6 +208,7 @@ enum { EXC_KERNEL, EXC_USER, EXC_DOUBLE, + EXC_DEBUG, EXC_MAX }; @@ -279,6 +302,11 @@ typedef struct XtensaConfig { uint32_t timerint[MAX_NCCOMPARE]; unsigned nextint; unsigned extint[MAX_NINTERRUPT]; + + unsigned debug_level; + unsigned nibreak; + unsigned ndbreak; + uint32_t clock_freq_khz; xtensa_tlb itlb; @@ -310,6 +338,9 @@ typedef struct CPUXtensaState { int exception_taken; + /* Watchpoints for DBREAK registers */ + CPUWatchpoint *cpu_watchpoint[MAX_NDBREAK]; + CPU_COMMON } CPUXtensaState; @@ -344,6 +375,8 @@ void xtensa_tlb_set_entry(CPUState *env, bool dtlb, int xtensa_get_physical_addr(CPUState *env, uint32_t vaddr, int is_write, int mmu_idx, uint32_t *paddr, uint32_t *page_size, unsigned *access); +void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUState *env); +void debug_exception_env(CPUState *new_env, uint32_t cause); #define XTENSA_OPTION_BIT(opt) (((uint64_t)1) << (opt)) @@ -409,6 +442,8 @@ static inline int cpu_mmu_index(CPUState *env) #define XTENSA_TBFLAG_RING_MASK 0x3 #define XTENSA_TBFLAG_EXCM 0x4 #define XTENSA_TBFLAG_LITBASE 0x8 +#define XTENSA_TBFLAG_DEBUG 0x10 +#define XTENSA_TBFLAG_ICOUNT 0x20 static inline void cpu_get_tb_cpu_state(CPUState *env, target_ulong *pc, target_ulong *cs_base, int *flags) @@ -424,6 +459,14 @@ static inline void cpu_get_tb_cpu_state(CPUState *env, target_ulong *pc, (env->sregs[LITBASE] & 1)) { *flags |= XTENSA_TBFLAG_LITBASE; } + if (xtensa_option_enabled(env->config, XTENSA_OPTION_DEBUG)) { + if (xtensa_get_cintlevel(env) < env->config->debug_level) { + *flags |= XTENSA_TBFLAG_DEBUG; + } + if (xtensa_get_cintlevel(env) < env->sregs[ICOUNTLEVEL]) { + *flags |= XTENSA_TBFLAG_ICOUNT; + } + } } #include "cpu-all.h" diff --git a/target-xtensa/helper.c b/target-xtensa/helper.c index 2a0cb1a562..2ef50d656e 100644 --- a/target-xtensa/helper.c +++ b/target-xtensa/helper.c @@ -44,6 +44,7 @@ void cpu_reset(CPUXtensaState *env) env->sregs[PS] = xtensa_option_enabled(env->config, XTENSA_OPTION_INTERRUPT) ? 0x1f : 0x10; env->sregs[VECBASE] = env->config->vecbase; + env->sregs[IBREAKENABLE] = 0; env->pending_irq_level = 0; reset_mmu(env); @@ -57,9 +58,44 @@ void xtensa_register_core(XtensaConfigList *node) xtensa_cores = node; } +static uint32_t check_hw_breakpoints(CPUState *env) +{ + unsigned i; + + for (i = 0; i < env->config->ndbreak; ++i) { + if (env->cpu_watchpoint[i] && + env->cpu_watchpoint[i]->flags & BP_WATCHPOINT_HIT) { + return DEBUGCAUSE_DB | (i << DEBUGCAUSE_DBNUM_SHIFT); + } + } + return 0; +} + +static CPUDebugExcpHandler *prev_debug_excp_handler; + +static void breakpoint_handler(CPUState *env) +{ + if (env->watchpoint_hit) { + if (env->watchpoint_hit->flags & BP_CPU) { + uint32_t cause; + + env->watchpoint_hit = NULL; + cause = check_hw_breakpoints(env); + if (cause) { + debug_exception_env(env, cause); + } + cpu_resume_from_signal(env, NULL); + } + } + if (prev_debug_excp_handler) { + prev_debug_excp_handler(env); + } +} + CPUXtensaState *cpu_xtensa_init(const char *cpu_model) { static int tcg_inited; + static int debug_handler_inited; CPUXtensaState *env; const XtensaConfig *config = NULL; XtensaConfigList *core = xtensa_cores; @@ -83,6 +119,12 @@ CPUXtensaState *cpu_xtensa_init(const char *cpu_model) xtensa_translate_init(); } + if (!debug_handler_inited && tcg_enabled()) { + debug_handler_inited = 1; + prev_debug_excp_handler = + cpu_set_debug_excp_handler(breakpoint_handler); + } + xtensa_irq_init(env); qemu_init_vcpu(env); return env; @@ -193,6 +235,7 @@ void do_interrupt(CPUState *env) case EXC_KERNEL: case EXC_USER: case EXC_DOUBLE: + case EXC_DEBUG: qemu_log_mask(CPU_LOG_INT, "%s(%d) " "pc = %08x, a0 = %08x, ps = %08x, ccount = %08x\n", __func__, env->exception_index, @@ -540,3 +583,70 @@ int xtensa_get_physical_addr(CPUState *env, return 0; } } + +static void dump_tlb(FILE *f, fprintf_function cpu_fprintf, + CPUState *env, bool dtlb) +{ + unsigned wi, ei; + const xtensa_tlb *conf = + dtlb ? &env->config->dtlb : &env->config->itlb; + unsigned (*attr_to_access)(uint32_t) = + xtensa_option_enabled(env->config, XTENSA_OPTION_MMU) ? + mmu_attr_to_access : region_attr_to_access; + + for (wi = 0; wi < conf->nways; ++wi) { + uint32_t sz = ~xtensa_tlb_get_addr_mask(env, dtlb, wi) + 1; + const char *sz_text; + bool print_header = true; + + if (sz >= 0x100000) { + sz >>= 20; + sz_text = "MB"; + } else { + sz >>= 10; + sz_text = "KB"; + } + + for (ei = 0; ei < conf->way_size[wi]; ++ei) { + const xtensa_tlb_entry *entry = + xtensa_tlb_get_entry(env, dtlb, wi, ei); + + if (entry->asid) { + unsigned access = attr_to_access(entry->attr); + + if (print_header) { + print_header = false; + cpu_fprintf(f, "Way %u (%d %s)\n", wi, sz, sz_text); + cpu_fprintf(f, + "\tVaddr Paddr ASID Attr RWX\n" + "\t---------- ---------- ---- ---- ---\n"); + } + cpu_fprintf(f, + "\t0x%08x 0x%08x 0x%02x 0x%02x %c%c%c\n", + entry->vaddr, + entry->paddr, + entry->asid, + entry->attr, + (access & PAGE_READ) ? 'R' : '-', + (access & PAGE_WRITE) ? 'W' : '-', + (access & PAGE_EXEC) ? 'X' : '-'); + } + } + } +} + +void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUState *env) +{ + if (xtensa_option_bits_enabled(env->config, + XTENSA_OPTION_BIT(XTENSA_OPTION_REGION_PROTECTION) | + XTENSA_OPTION_BIT(XTENSA_OPTION_REGION_TRANSLATION) | + XTENSA_OPTION_BIT(XTENSA_OPTION_MMU))) { + + cpu_fprintf(f, "ITLB:\n"); + dump_tlb(f, cpu_fprintf, env, false); + cpu_fprintf(f, "\nDTLB:\n"); + dump_tlb(f, cpu_fprintf, env, true); + } else { + cpu_fprintf(f, "No TLB for this CPU core\n"); + } +} diff --git a/target-xtensa/helpers.h b/target-xtensa/helpers.h index 09ab3325c9..48a741e46d 100644 --- a/target-xtensa/helpers.h +++ b/target-xtensa/helpers.h @@ -3,6 +3,8 @@ DEF_HELPER_1(exception, void, i32) DEF_HELPER_2(exception_cause, void, i32, i32) DEF_HELPER_3(exception_cause_vaddr, void, i32, i32, i32) +DEF_HELPER_2(debug_exception, void, i32, i32) + DEF_HELPER_1(nsa, i32, i32) DEF_HELPER_1(nsau, i32, i32) DEF_HELPER_1(wsr_windowbase, void, i32) @@ -29,4 +31,9 @@ DEF_HELPER_2(itlb, void, i32, i32) DEF_HELPER_2(ptlb, i32, i32, i32) DEF_HELPER_3(wtlb, void, i32, i32, i32) +DEF_HELPER_1(wsr_ibreakenable, void, i32) +DEF_HELPER_2(wsr_ibreaka, void, i32, i32) +DEF_HELPER_2(wsr_dbreaka, void, i32, i32) +DEF_HELPER_2(wsr_dbreakc, void, i32, i32) + #include "def-helper.h" diff --git a/target-xtensa/op_helper.c b/target-xtensa/op_helper.c index 0605611031..e184cf64f0 100644 --- a/target-xtensa/op_helper.c +++ b/target-xtensa/op_helper.c @@ -134,6 +134,27 @@ void HELPER(exception_cause_vaddr)(uint32_t pc, uint32_t cause, uint32_t vaddr) HELPER(exception_cause)(pc, cause); } +void debug_exception_env(CPUState *new_env, uint32_t cause) +{ + if (xtensa_get_cintlevel(new_env) < new_env->config->debug_level) { + env = new_env; + HELPER(debug_exception)(env->pc, cause); + } +} + +void HELPER(debug_exception)(uint32_t pc, uint32_t cause) +{ + unsigned level = env->config->debug_level; + + env->pc = pc; + env->sregs[DEBUGCAUSE] = cause; + env->sregs[EPC1 + level - 1] = pc; + env->sregs[EPS2 + level - 2] = env->sregs[PS]; + env->sregs[PS] = (env->sregs[PS] & ~PS_INTLEVEL) | PS_EXCM | + (level << PS_INTLEVEL_SHIFT); + HELPER(exception)(EXC_DEBUG); +} + uint32_t HELPER(nsa)(uint32_t v) { if (v & 0x80000000) { @@ -662,3 +683,82 @@ void HELPER(wtlb)(uint32_t p, uint32_t v, uint32_t dtlb) split_tlb_entry_spec(v, dtlb, &vpn, &wi, &ei); xtensa_tlb_set_entry(env, dtlb, wi, ei, vpn, p); } + + +void HELPER(wsr_ibreakenable)(uint32_t v) +{ + uint32_t change = v ^ env->sregs[IBREAKENABLE]; + unsigned i; + + for (i = 0; i < env->config->nibreak; ++i) { + if (change & (1 << i)) { + tb_invalidate_phys_page_range( + env->sregs[IBREAKA + i], env->sregs[IBREAKA + i] + 1, 0); + } + } + env->sregs[IBREAKENABLE] = v & ((1 << env->config->nibreak) - 1); +} + +void HELPER(wsr_ibreaka)(uint32_t i, uint32_t v) +{ + if (env->sregs[IBREAKENABLE] & (1 << i) && env->sregs[IBREAKA + i] != v) { + tb_invalidate_phys_page_range( + env->sregs[IBREAKA + i], env->sregs[IBREAKA + i] + 1, 0); + tb_invalidate_phys_page_range(v, v + 1, 0); + } + env->sregs[IBREAKA + i] = v; +} + +static void set_dbreak(unsigned i, uint32_t dbreaka, uint32_t dbreakc) +{ + int flags = BP_CPU | BP_STOP_BEFORE_ACCESS; + uint32_t mask = dbreakc | ~DBREAKC_MASK; + + if (env->cpu_watchpoint[i]) { + cpu_watchpoint_remove_by_ref(env, env->cpu_watchpoint[i]); + } + if (dbreakc & DBREAKC_SB) { + flags |= BP_MEM_WRITE; + } + if (dbreakc & DBREAKC_LB) { + flags |= BP_MEM_READ; + } + /* contiguous mask after inversion is one less than some power of 2 */ + if ((~mask + 1) & ~mask) { + qemu_log("DBREAKC mask is not contiguous: 0x%08x\n", dbreakc); + /* cut mask after the first zero bit */ + mask = 0xffffffff << (32 - clo32(mask)); + } + if (cpu_watchpoint_insert(env, dbreaka & mask, ~mask + 1, + flags, &env->cpu_watchpoint[i])) { + env->cpu_watchpoint[i] = NULL; + qemu_log("Failed to set data breakpoint at 0x%08x/%d\n", + dbreaka & mask, ~mask + 1); + } +} + +void HELPER(wsr_dbreaka)(uint32_t i, uint32_t v) +{ + uint32_t dbreakc = env->sregs[DBREAKC + i]; + + if ((dbreakc & DBREAKC_SB_LB) && + env->sregs[DBREAKA + i] != v) { + set_dbreak(i, v, dbreakc); + } + env->sregs[DBREAKA + i] = v; +} + +void HELPER(wsr_dbreakc)(uint32_t i, uint32_t v) +{ + if ((env->sregs[DBREAKC + i] ^ v) & (DBREAKC_SB_LB | DBREAKC_MASK)) { + if (v & DBREAKC_SB_LB) { + set_dbreak(i, env->sregs[DBREAKA + i], v); + } else { + if (env->cpu_watchpoint[i]) { + cpu_watchpoint_remove_by_ref(env, env->cpu_watchpoint[i]); + env->cpu_watchpoint[i] = NULL; + } + } + } + env->sregs[DBREAKC + i] = v; +} diff --git a/target-xtensa/overlay_tool.h b/target-xtensa/overlay_tool.h index df19cc96ea..a3a5650fb0 100644 --- a/target-xtensa/overlay_tool.h +++ b/target-xtensa/overlay_tool.h @@ -114,6 +114,7 @@ [EXC_KERNEL] = XCHAL_KERNEL_VECTOR_VADDR, \ [EXC_USER] = XCHAL_USER_VECTOR_VADDR, \ [EXC_DOUBLE] = XCHAL_DOUBLEEXC_VECTOR_VADDR, \ + [EXC_DEBUG] = XCHAL_DEBUG_VECTOR_VADDR, \ } #define INTERRUPT_VECTORS { \ @@ -251,6 +252,8 @@ .nextint = XCHAL_NUM_EXTINTERRUPTS, \ .extint = EXTINTS +#if XCHAL_HAVE_PTP_MMU + #define TLB_TEMPLATE(ways, refill_way_size, way56) { \ .nways = ways, \ .way_size = { \ @@ -268,11 +271,23 @@ #define DTLB(varway56) \ TLB_TEMPLATE(10, 1 << XCHAL_DTLB_ARF_ENTRIES_LOG2, varway56) -#if XCHAL_HAVE_PTP_MMU #define TLB_SECTION \ .itlb = ITLB(XCHAL_HAVE_SPANNING_WAY), \ .dtlb = DTLB(XCHAL_HAVE_SPANNING_WAY) -#else + +#elif XCHAL_HAVE_XLT_CACHEATTR || XCHAL_HAVE_MIMIC_CACHEATTR + +#define TLB_TEMPLATE { \ + .nways = 1, \ + .way_size = { \ + 8, \ + } \ + } + +#define TLB_SECTION \ + .itlb = TLB_TEMPLATE, \ + .dtlb = TLB_TEMPLATE + #endif #if (defined(TARGET_WORDS_BIGENDIAN) != 0) == (XCHAL_HAVE_BE != 0) @@ -288,6 +303,10 @@ #define REGISTER_CORE(core) #endif +#define DEBUG_SECTION \ + .debug_level = XCHAL_DEBUGLEVEL, \ + .nibreak = XCHAL_NUM_IBREAK, \ + .ndbreak = XCHAL_NUM_DBREAK #if XCHAL_NUM_INTLEVELS + XCHAL_HAVE_NMI + 1 <= 2 #define XCHAL_INTLEVEL2_VECTOR_VADDR 0 diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c index c81450d1a5..9e8e20a904 100644 --- a/target-xtensa/translate.c +++ b/target-xtensa/translate.c @@ -61,6 +61,10 @@ typedef struct DisasContext { uint32_t ccount_delta; unsigned used_window; + + bool debug; + bool icount; + TCGv_i32 next_icount; } DisasContext; static TCGv_ptr cpu_env; @@ -91,6 +95,13 @@ static const char * const sregnames[256] = { [RASID] = "RASID", [ITLBCFG] = "ITLBCFG", [DTLBCFG] = "DTLBCFG", + [IBREAKENABLE] = "IBREAKENABLE", + [IBREAKA] = "IBREAKA0", + [IBREAKA + 1] = "IBREAKA1", + [DBREAKA] = "DBREAKA0", + [DBREAKA + 1] = "DBREAKA1", + [DBREAKC] = "DBREAKC0", + [DBREAKC + 1] = "DBREAKC1", [EPC1] = "EPC1", [EPC1 + 1] = "EPC2", [EPC1 + 2] = "EPC3", @@ -119,8 +130,11 @@ static const char * const sregnames[256] = { [PS] = "PS", [VECBASE] = "VECBASE", [EXCCAUSE] = "EXCCAUSE", + [DEBUGCAUSE] = "DEBUGCAUSE", [CCOUNT] = "CCOUNT", [PRID] = "PRID", + [ICOUNT] = "ICOUNT", + [ICOUNTLEVEL] = "ICOUNTLEVEL", [EXCVADDR] = "EXCVADDR", [CCOMPARE] = "CCOMPARE0", [CCOMPARE + 1] = "CCOMPARE1", @@ -283,6 +297,19 @@ static void gen_exception_cause_vaddr(DisasContext *dc, uint32_t cause, tcg_temp_free(tcause); } +static void gen_debug_exception(DisasContext *dc, uint32_t cause) +{ + TCGv_i32 tpc = tcg_const_i32(dc->pc); + TCGv_i32 tcause = tcg_const_i32(cause); + gen_advance_ccount(dc); + gen_helper_debug_exception(tpc, tcause); + tcg_temp_free(tpc); + tcg_temp_free(tcause); + if (cause & (DEBUGCAUSE_IB | DEBUGCAUSE_BI | DEBUGCAUSE_BN)) { + dc->is_jmp = DISAS_UPDATE; + } +} + static void gen_check_privilege(DisasContext *dc) { if (dc->cring) { @@ -294,10 +321,13 @@ static void gen_check_privilege(DisasContext *dc) static void gen_jump_slot(DisasContext *dc, TCGv dest, int slot) { tcg_gen_mov_i32(cpu_pc, dest); + gen_advance_ccount(dc); + if (dc->icount) { + tcg_gen_mov_i32(cpu_SR[ICOUNT], dc->next_icount); + } if (dc->singlestep_enabled) { gen_exception(dc, EXCP_DEBUG); } else { - gen_advance_ccount(dc); if (slot >= 0) { tcg_gen_goto_tb(slot); tcg_gen_exit_tb((tcg_target_long)dc->tb + slot); @@ -492,6 +522,46 @@ static void gen_wsr_tlbcfg(DisasContext *dc, uint32_t sr, TCGv_i32 v) tcg_gen_andi_i32(cpu_SR[sr], v, 0x01130000); } +static void gen_wsr_ibreakenable(DisasContext *dc, uint32_t sr, TCGv_i32 v) +{ + gen_helper_wsr_ibreakenable(v); + gen_jumpi_check_loop_end(dc, 0); +} + +static void gen_wsr_ibreaka(DisasContext *dc, uint32_t sr, TCGv_i32 v) +{ + unsigned id = sr - IBREAKA; + + if (id < dc->config->nibreak) { + TCGv_i32 tmp = tcg_const_i32(id); + gen_helper_wsr_ibreaka(tmp, v); + tcg_temp_free(tmp); + gen_jumpi_check_loop_end(dc, 0); + } +} + +static void gen_wsr_dbreaka(DisasContext *dc, uint32_t sr, TCGv_i32 v) +{ + unsigned id = sr - DBREAKA; + + if (id < dc->config->ndbreak) { + TCGv_i32 tmp = tcg_const_i32(id); + gen_helper_wsr_dbreaka(tmp, v); + tcg_temp_free(tmp); + } +} + +static void gen_wsr_dbreakc(DisasContext *dc, uint32_t sr, TCGv_i32 v) +{ + unsigned id = sr - DBREAKC; + + if (id < dc->config->ndbreak) { + TCGv_i32 tmp = tcg_const_i32(id); + gen_helper_wsr_dbreakc(tmp, v); + tcg_temp_free(tmp); + } +} + static void gen_wsr_intset(DisasContext *dc, uint32_t sr, TCGv_i32 v) { tcg_gen_andi_i32(cpu_SR[sr], v, @@ -535,10 +605,30 @@ static void gen_wsr_ps(DisasContext *dc, uint32_t sr, TCGv_i32 v) gen_jumpi_check_loop_end(dc, -1); } +static void gen_wsr_debugcause(DisasContext *dc, uint32_t sr, TCGv_i32 v) +{ +} + static void gen_wsr_prid(DisasContext *dc, uint32_t sr, TCGv_i32 v) { } +static void gen_wsr_icount(DisasContext *dc, uint32_t sr, TCGv_i32 v) +{ + if (dc->icount) { + tcg_gen_mov_i32(dc->next_icount, v); + } else { + tcg_gen_mov_i32(cpu_SR[sr], v); + } +} + +static void gen_wsr_icountlevel(DisasContext *dc, uint32_t sr, TCGv_i32 v) +{ + tcg_gen_andi_i32(cpu_SR[sr], v, 0xf); + /* This can change tb->flags, so exit tb */ + gen_jumpi_check_loop_end(dc, -1); +} + static void gen_wsr_ccompare(DisasContext *dc, uint32_t sr, TCGv_i32 v) { uint32_t id = sr - CCOMPARE; @@ -567,11 +657,21 @@ static void gen_wsr(DisasContext *dc, uint32_t sr, TCGv_i32 s) [RASID] = gen_wsr_rasid, [ITLBCFG] = gen_wsr_tlbcfg, [DTLBCFG] = gen_wsr_tlbcfg, + [IBREAKENABLE] = gen_wsr_ibreakenable, + [IBREAKA] = gen_wsr_ibreaka, + [IBREAKA + 1] = gen_wsr_ibreaka, + [DBREAKA] = gen_wsr_dbreaka, + [DBREAKA + 1] = gen_wsr_dbreaka, + [DBREAKC] = gen_wsr_dbreakc, + [DBREAKC + 1] = gen_wsr_dbreakc, [INTSET] = gen_wsr_intset, [INTCLEAR] = gen_wsr_intclear, [INTENABLE] = gen_wsr_intenable, [PS] = gen_wsr_ps, + [DEBUGCAUSE] = gen_wsr_debugcause, [PRID] = gen_wsr_prid, + [ICOUNT] = gen_wsr_icount, + [ICOUNTLEVEL] = gen_wsr_icountlevel, [CCOMPARE] = gen_wsr_ccompare, [CCOMPARE + 1] = gen_wsr_ccompare, [CCOMPARE + 2] = gen_wsr_ccompare, @@ -749,7 +849,7 @@ static void disas_xtensa_insn(DisasContext *dc) uint8_t b0 = ldub_code(dc->pc); uint8_t b1 = ldub_code(dc->pc + 1); - uint8_t b2 = ldub_code(dc->pc + 2); + uint8_t b2 = 0; static const uint32_t B4CONST[] = { 0xffffffff, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 16, 32, 64, 128, 256 @@ -764,6 +864,7 @@ static void disas_xtensa_insn(DisasContext *dc) HAS_OPTION(XTENSA_OPTION_CODE_DENSITY); } else { dc->next_pc = dc->pc + 3; + b2 = ldub_code(dc->pc + 2); } switch (OP0) { @@ -968,8 +1069,10 @@ static void disas_xtensa_insn(DisasContext *dc) break; case 4: /*BREAKx*/ - HAS_OPTION(XTENSA_OPTION_EXCEPTION); - TBD(); + HAS_OPTION(XTENSA_OPTION_DEBUG); + if (dc->debug) { + gen_debug_exception(dc, DEBUGCAUSE_BI); + } break; case 5: /*SYSCALLx*/ @@ -2349,7 +2452,10 @@ static void disas_xtensa_insn(DisasContext *dc) break; case 2: /*BREAK.Nn*/ - TBD(); + HAS_OPTION(XTENSA_OPTION_DEBUG); + if (dc->debug) { + gen_debug_exception(dc, DEBUGCAUSE_BN); + } break; case 3: /*NOP.Nn*/ @@ -2402,6 +2508,19 @@ static void check_breakpoint(CPUState *env, DisasContext *dc) } } +static void gen_ibreak_check(CPUState *env, DisasContext *dc) +{ + unsigned i; + + for (i = 0; i < dc->config->nibreak; ++i) { + if ((env->sregs[IBREAKENABLE] & (1 << i)) && + env->sregs[IBREAKA + i] == dc->pc) { + gen_debug_exception(dc, DEBUGCAUSE_IB); + break; + } + } +} + static void gen_intermediate_code_internal( CPUState *env, TranslationBlock *tb, int search_pc) { @@ -2428,10 +2547,15 @@ static void gen_intermediate_code_internal( dc.lend = env->sregs[LEND]; dc.is_jmp = DISAS_NEXT; dc.ccount_delta = 0; + dc.debug = tb->flags & XTENSA_TBFLAG_DEBUG; + dc.icount = tb->flags & XTENSA_TBFLAG_ICOUNT; init_litbase(&dc); init_sar_tracker(&dc); reset_used_window(&dc); + if (dc.icount) { + dc.next_icount = tcg_temp_local_new_i32(); + } gen_icount_start(); @@ -2467,8 +2591,27 @@ static void gen_intermediate_code_internal( gen_io_start(); } + if (dc.icount) { + int label = gen_new_label(); + + tcg_gen_addi_i32(dc.next_icount, cpu_SR[ICOUNT], 1); + tcg_gen_brcondi_i32(TCG_COND_NE, dc.next_icount, 0, label); + tcg_gen_mov_i32(dc.next_icount, cpu_SR[ICOUNT]); + if (dc.debug) { + gen_debug_exception(&dc, DEBUGCAUSE_IC); + } + gen_set_label(label); + } + + if (dc.debug) { + gen_ibreak_check(env, &dc); + } + disas_xtensa_insn(&dc); ++insn_count; + if (dc.icount) { + tcg_gen_mov_i32(cpu_SR[ICOUNT], dc.next_icount); + } if (env->singlestep_enabled) { tcg_gen_movi_i32(cpu_pc, dc.pc); gen_exception(&dc, EXCP_DEBUG); @@ -2481,6 +2624,9 @@ static void gen_intermediate_code_internal( reset_litbase(&dc); reset_sar_tracker(&dc); + if (dc.icount) { + tcg_temp_free(dc.next_icount); + } if (tb->cflags & CF_LAST_IO) { gen_io_end(); diff --git a/tests/tcg/xtensa/Makefile b/tests/tcg/xtensa/Makefile index 8713af16eb..7e1e619d23 100644 --- a/tests/tcg/xtensa/Makefile +++ b/tests/tcg/xtensa/Makefile @@ -23,6 +23,7 @@ CRT = crt.o vectors.o TESTCASES += test_b.tst TESTCASES += test_bi.tst #TESTCASES += test_boolean.tst +TESTCASES += test_break.tst TESTCASES += test_bz.tst TESTCASES += test_clamps.tst TESTCASES += test_fail.tst diff --git a/tests/tcg/xtensa/test_break.S b/tests/tcg/xtensa/test_break.S new file mode 100644 index 0000000000..8a8db8033b --- /dev/null +++ b/tests/tcg/xtensa/test_break.S @@ -0,0 +1,223 @@ +.include "macros.inc" + +#define debug_level 6 +#define debug_vector level6 + +test_suite break + +test break + set_vector debug_vector, 0 + rsil a2, debug_level + _break 0, 0 + + set_vector debug_vector, 2f + rsil a2, debug_level - 1 +1: + _break 0, 0 + test_fail +2: + rsr a2, ps + movi a3, 0x1f + and a2, a2, a3 + movi a3, 0x10 | debug_level + assert eq, a2, a3 + rsr a2, epc6 + movi a3, 1b + assert eq, a2, a3 + rsr a2, debugcause + movi a3, 0x8 + assert eq, a2, a3 +test_end + +test breakn + set_vector debug_vector, 0 + rsil a2, debug_level + _break.n 0 + + set_vector debug_vector, 2f + rsil a2, debug_level - 1 +1: + _break.n 0 + test_fail +2: + rsr a2, ps + movi a3, 0x1f + and a2, a2, a3 + movi a3, 0x10 | debug_level + assert eq, a2, a3 + rsr a2, epc6 + movi a3, 1b + assert eq, a2, a3 + rsr a2, debugcause + movi a3, 0x10 + assert eq, a2, a3 +test_end + +test ibreak + set_vector debug_vector, 0 + rsil a2, debug_level + movi a2, 1f + wsr a2, ibreaka0 + movi a2, 1 + wsr a2, ibreakenable + isync +1: + rsil a2, debug_level - 1 + movi a2, 1f + wsr a2, ibreaka0 + movi a2, 0 + wsr a2, ibreakenable + isync +1: + set_vector debug_vector, 2f + movi a2, 1f + wsr a2, ibreaka0 + movi a2, 1 + wsr a2, ibreakenable + isync +1: + test_fail +2: + rsr a2, ps + movi a3, 0x1f + and a2, a2, a3 + movi a3, 0x10 | debug_level + assert eq, a2, a3 + rsr a2, epc6 + movi a3, 1b + assert eq, a2, a3 + rsr a2, debugcause + movi a3, 0x2 + assert eq, a2, a3 +test_end + +test ibreak_priority + set_vector debug_vector, 2f + rsil a2, debug_level - 1 + movi a2, 1f + wsr a2, ibreaka0 + movi a2, 1 + wsr a2, ibreakenable + isync +1: + break 0, 0 + test_fail +2: + rsr a2, debugcause + movi a3, 0x2 + assert eq, a2, a3 +test_end + +test icount + set_vector debug_vector, 2f + rsil a2, debug_level - 1 + movi a2, -2 + wsr a2, icount + movi a2, 1 + wsr a2, icountlevel + isync + rsil a2, 0 + nop +1: + break 0, 0 + test_fail +2: + movi a2, 0 + wsr a2, icountlevel + rsr a2, epc6 + movi a3, 1b + assert eq, a2, a3 + rsr a2, debugcause + movi a3, 0x1 + assert eq, a2, a3 +test_end + +.macro check_dbreak dr + rsr a2, epc6 + movi a3, 1b + assert eq, a2, a3 + rsr a2, debugcause + movi a3, 0x4 | (\dr << 8) + assert eq, a2, a3 + movi a2, 0 + wsr a2, dbreakc\dr +.endm + +.macro dbreak_test dr, ctl, break, access, op + set_vector debug_vector, 2f + rsil a2, debug_level - 1 + movi a2, \ctl + wsr a2, dbreakc\dr + movi a2, \break + wsr a2, dbreaka\dr + movi a2, \access + isync +1: + \op a3, a2, 0 + test_fail +2: + check_dbreak \dr + reset_ps +.endm + +test dbreak_exact + dbreak_test 0, 0x4000003f, 0xd000007f, 0xd000007f, l8ui + dbreak_test 1, 0x4000003e, 0xd000007e, 0xd000007e, l16ui + dbreak_test 0, 0x4000003c, 0xd000007c, 0xd000007c, l32i + + dbreak_test 1, 0x8000003f, 0xd000007f, 0xd000007f, s8i + dbreak_test 0, 0x8000003e, 0xd000007e, 0xd000007e, s16i + dbreak_test 1, 0x8000003c, 0xd000007c, 0xd000007c, s32i +test_end + +test dbreak_overlap + dbreak_test 0, 0x4000003f, 0xd000007d, 0xd000007c, l16ui + dbreak_test 1, 0x4000003f, 0xd000007d, 0xd000007c, l32i + + dbreak_test 0, 0x4000003e, 0xd000007e, 0xd000007f, l8ui + dbreak_test 1, 0x4000003e, 0xd000007e, 0xd000007c, l32i + + dbreak_test 0, 0x4000003c, 0xd000007c, 0xd000007d, l8ui + dbreak_test 1, 0x4000003c, 0xd000007c, 0xd000007c, l16ui + + dbreak_test 0, 0x40000038, 0xd0000078, 0xd000007b, l8ui + dbreak_test 1, 0x40000038, 0xd0000078, 0xd000007a, l16ui + dbreak_test 0, 0x40000038, 0xd0000078, 0xd000007c, l32i + + dbreak_test 1, 0x40000030, 0xd0000070, 0xd0000075, l8ui + dbreak_test 0, 0x40000030, 0xd0000070, 0xd0000076, l16ui + dbreak_test 1, 0x40000030, 0xd0000070, 0xd0000078, l32i + + dbreak_test 0, 0x40000020, 0xd0000060, 0xd000006f, l8ui + dbreak_test 1, 0x40000020, 0xd0000060, 0xd0000070, l16ui + dbreak_test 0, 0x40000020, 0xd0000060, 0xd0000074, l32i + + + dbreak_test 0, 0x8000003f, 0xd000007d, 0xd000007c, s16i + dbreak_test 1, 0x8000003f, 0xd000007d, 0xd000007c, s32i + + dbreak_test 0, 0x8000003e, 0xd000007e, 0xd000007f, s8i + dbreak_test 1, 0x8000003e, 0xd000007e, 0xd000007c, s32i + + dbreak_test 0, 0x8000003c, 0xd000007c, 0xd000007d, s8i + dbreak_test 1, 0x8000003c, 0xd000007c, 0xd000007c, s16i + + dbreak_test 0, 0x80000038, 0xd0000078, 0xd000007b, s8i + dbreak_test 1, 0x80000038, 0xd0000078, 0xd000007a, s16i + dbreak_test 0, 0x80000038, 0xd0000078, 0xd000007c, s32i + + dbreak_test 1, 0x80000030, 0xd0000070, 0xd0000075, s8i + dbreak_test 0, 0x80000030, 0xd0000070, 0xd0000076, s16i + dbreak_test 1, 0x80000030, 0xd0000070, 0xd0000078, s32i + + dbreak_test 0, 0x80000020, 0xd0000060, 0xd000006f, s8i + dbreak_test 1, 0x80000020, 0xd0000060, 0xd0000070, s16i + dbreak_test 0, 0x80000020, 0xd0000060, 0xd0000074, s32i +test_end + +test dbreak_invalid + dbreak_test 0, 0x40000030, 0xd0000071, 0xd0000070, l16ui + dbreak_test 1, 0x40000035, 0xd0000072, 0xd0000070, l32i +test_end + +test_suite_end @@ -392,6 +392,14 @@ static void xen_set_memory(struct MemoryListener *listener, } } +static void xen_begin(MemoryListener *listener) +{ +} + +static void xen_commit(MemoryListener *listener) +{ +} + static void xen_region_add(MemoryListener *listener, MemoryRegionSection *section) { @@ -404,6 +412,11 @@ static void xen_region_del(MemoryListener *listener, xen_set_memory(listener, section, false); } +static void xen_region_nop(MemoryListener *listener, + MemoryRegionSection *section) +{ +} + static void xen_sync_dirty_bitmap(XenIOState *state, target_phys_addr_t start_addr, ram_addr_t size) @@ -485,14 +498,32 @@ static void xen_log_global_stop(MemoryListener *listener) { } +static void xen_eventfd_add(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + +static void xen_eventfd_del(MemoryListener *listener, + MemoryRegionSection *section, + bool match_data, uint64_t data, int fd) +{ +} + static MemoryListener xen_memory_listener = { + .begin = xen_begin, + .commit = xen_commit, .region_add = xen_region_add, .region_del = xen_region_del, + .region_nop = xen_region_nop, .log_start = xen_log_start, .log_stop = xen_log_stop, .log_sync = xen_log_sync, .log_global_start = xen_log_global_start, .log_global_stop = xen_log_global_stop, + .eventfd_add = xen_eventfd_add, + .eventfd_del = xen_eventfd_del, + .priority = 10, }; /* VCPU Operations, MMIO, IO ring ... */ @@ -975,7 +1006,7 @@ int xen_hvm_init(void) state->memory_listener = xen_memory_listener; QLIST_INIT(&state->physmap); - memory_listener_register(&state->memory_listener); + memory_listener_register(&state->memory_listener, get_system_memory()); state->log_for_dirtybit = NULL; /* Initialize backend core & drivers */ |