aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--exec-obsolete.h5
-rw-r--r--exec.c875
-rw-r--r--hw/vhost.c33
-rw-r--r--ioport.c28
-rw-r--r--ioport.h1
-rw-r--r--kvm-all.c97
-rw-r--r--memory.c328
-rw-r--r--memory.h26
-rw-r--r--xen-all.c33
9 files changed, 912 insertions, 514 deletions
diff --git a/exec-obsolete.h b/exec-obsolete.h
index 94c23d0951..4dbe4768aa 100644
--- a/exec-obsolete.h
+++ b/exec-obsolete.h
@@ -37,7 +37,7 @@ void cpu_unregister_io_memory(int table_address);
struct MemoryRegionSection;
void cpu_register_physical_memory_log(struct MemoryRegionSection *section,
- bool readable, bool readonly);
+ bool readonly);
void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
@@ -121,6 +121,9 @@ static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start,
void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
int dirty_flags);
+
+extern const IORangeOps memory_region_iorange_ops;
+
#endif
#endif
diff --git a/exec.c b/exec.c
index b81677ade9..0c93b2691f 100644
--- a/exec.c
+++ b/exec.c
@@ -160,29 +160,21 @@ typedef struct PageDesc {
#define L2_BITS 10
#define L2_SIZE (1 << L2_BITS)
+#define P_L2_LEVELS \
+ (((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - 1) / L2_BITS) + 1)
+
/* The bits remaining after N lower levels of page tables. */
-#define P_L1_BITS_REM \
- ((TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
#define V_L1_BITS_REM \
((L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS) % L2_BITS)
-/* Size of the L1 page table. Avoid silly small sizes. */
-#if P_L1_BITS_REM < 4
-#define P_L1_BITS (P_L1_BITS_REM + L2_BITS)
-#else
-#define P_L1_BITS P_L1_BITS_REM
-#endif
-
#if V_L1_BITS_REM < 4
#define V_L1_BITS (V_L1_BITS_REM + L2_BITS)
#else
#define V_L1_BITS V_L1_BITS_REM
#endif
-#define P_L1_SIZE ((target_phys_addr_t)1 << P_L1_BITS)
#define V_L1_SIZE ((target_ulong)1 << V_L1_BITS)
-#define P_L1_SHIFT (TARGET_PHYS_ADDR_SPACE_BITS - TARGET_PAGE_BITS - P_L1_BITS)
#define V_L1_SHIFT (L1_MAP_ADDR_SPACE_BITS - TARGET_PAGE_BITS - V_L1_BITS)
unsigned long qemu_real_host_page_size;
@@ -194,15 +186,27 @@ unsigned long qemu_host_page_mask;
static void *l1_map[V_L1_SIZE];
#if !defined(CONFIG_USER_ONLY)
-typedef struct PhysPageDesc {
- /* offset in host memory of the page + io_index in the low bits */
- ram_addr_t phys_offset;
- ram_addr_t region_offset;
-} PhysPageDesc;
+typedef struct PhysPageEntry PhysPageEntry;
+
+static MemoryRegionSection *phys_sections;
+static unsigned phys_sections_nb, phys_sections_nb_alloc;
+static uint16_t phys_section_unassigned;
+
+struct PhysPageEntry {
+ uint16_t is_leaf : 1;
+ /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
+ uint16_t ptr : 15;
+};
+
+/* Simple allocator for PhysPageEntry nodes */
+static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
+static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
+
+#define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
/* This is a multi-level map on the physical address space.
- The bottom level has pointers to PhysPageDesc. */
-static void *l1_phys_map[P_L1_SIZE];
+ The bottom level has pointers to MemoryRegionSections. */
+static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
static void io_mem_init(void);
static void memory_map_init(void);
@@ -398,59 +402,112 @@ static inline PageDesc *page_find(tb_page_addr_t index)
}
#if !defined(CONFIG_USER_ONLY)
-static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
+
+static void phys_map_node_reserve(unsigned nodes)
{
- PhysPageDesc *pd;
- void **lp;
- int i;
+ if (phys_map_nodes_nb + nodes > phys_map_nodes_nb_alloc) {
+ typedef PhysPageEntry Node[L2_SIZE];
+ phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc * 2, 16);
+ phys_map_nodes_nb_alloc = MAX(phys_map_nodes_nb_alloc,
+ phys_map_nodes_nb + nodes);
+ phys_map_nodes = g_renew(Node, phys_map_nodes,
+ phys_map_nodes_nb_alloc);
+ }
+}
- /* Level 1. Always allocated. */
- lp = l1_phys_map + ((index >> P_L1_SHIFT) & (P_L1_SIZE - 1));
+static uint16_t phys_map_node_alloc(void)
+{
+ unsigned i;
+ uint16_t ret;
- /* Level 2..N-1. */
- for (i = P_L1_SHIFT / L2_BITS - 1; i > 0; i--) {
- void **p = *lp;
- if (p == NULL) {
- if (!alloc) {
- return NULL;
- }
- *lp = p = g_malloc0(sizeof(void *) * L2_SIZE);
- }
- lp = p + ((index >> (i * L2_BITS)) & (L2_SIZE - 1));
+ ret = phys_map_nodes_nb++;
+ assert(ret != PHYS_MAP_NODE_NIL);
+ assert(ret != phys_map_nodes_nb_alloc);
+ for (i = 0; i < L2_SIZE; ++i) {
+ phys_map_nodes[ret][i].is_leaf = 0;
+ phys_map_nodes[ret][i].ptr = PHYS_MAP_NODE_NIL;
}
+ return ret;
+}
- pd = *lp;
- if (pd == NULL) {
- int i;
- int first_index = index & ~(L2_SIZE - 1);
+static void phys_map_nodes_reset(void)
+{
+ phys_map_nodes_nb = 0;
+}
- if (!alloc) {
- return NULL;
- }
- *lp = pd = g_malloc(sizeof(PhysPageDesc) * L2_SIZE);
+static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index,
+ target_phys_addr_t *nb, uint16_t leaf,
+ int level)
+{
+ PhysPageEntry *p;
+ int i;
+ target_phys_addr_t step = (target_phys_addr_t)1 << (level * L2_BITS);
+
+ if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
+ lp->ptr = phys_map_node_alloc();
+ p = phys_map_nodes[lp->ptr];
+ if (level == 0) {
+ for (i = 0; i < L2_SIZE; i++) {
+ p[i].is_leaf = 1;
+ p[i].ptr = phys_section_unassigned;
+ }
+ }
+ } else {
+ p = phys_map_nodes[lp->ptr];
+ }
+ lp = &p[(*index >> (level * L2_BITS)) & (L2_SIZE - 1)];
- for (i = 0; i < L2_SIZE; i++) {
- pd[i].phys_offset = io_mem_unassigned.ram_addr;
- pd[i].region_offset = (first_index + i) << TARGET_PAGE_BITS;
+ while (*nb && lp < &p[L2_SIZE]) {
+ if ((*index & (step - 1)) == 0 && *nb >= step) {
+ lp->is_leaf = true;
+ lp->ptr = leaf;
+ *index += step;
+ *nb -= step;
+ } else {
+ phys_page_set_level(lp, index, nb, leaf, level - 1);
}
+ ++lp;
}
+}
- return pd + (index & (L2_SIZE - 1));
+static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb,
+ uint16_t leaf)
+{
+ /* Wildly overreserve - it doesn't matter much. */
+ phys_map_node_reserve(3 * P_L2_LEVELS);
+
+ phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
}
-static inline PhysPageDesc phys_page_find(target_phys_addr_t index)
+static MemoryRegionSection phys_page_find(target_phys_addr_t index)
{
- PhysPageDesc *p = phys_page_find_alloc(index, 0);
+ PhysPageEntry lp = phys_map;
+ PhysPageEntry *p;
+ int i;
+ MemoryRegionSection section;
+ target_phys_addr_t delta;
+ uint16_t s_index = phys_section_unassigned;
- if (p) {
- return *p;
- } else {
- return (PhysPageDesc) {
- .phys_offset = io_mem_unassigned.ram_addr,
- .region_offset = index << TARGET_PAGE_BITS,
- };
+ for (i = P_L2_LEVELS - 1; i >= 0 && !lp.is_leaf; i--) {
+ if (lp.ptr == PHYS_MAP_NODE_NIL) {
+ goto not_found;
+ }
+ p = phys_map_nodes[lp.ptr];
+ lp = p[(index >> (i * L2_BITS)) & (L2_SIZE - 1)];
}
+
+ s_index = lp.ptr;
+not_found:
+ section = phys_sections[s_index];
+ index <<= TARGET_PAGE_BITS;
+ assert(section.offset_within_address_space <= index
+ && index <= section.offset_within_address_space + section.size-1);
+ delta = index - section.offset_within_address_space;
+ section.offset_within_address_space += delta;
+ section.offset_within_region += delta;
+ section.size -= delta;
+ return section;
}
static void tlb_protect_code(ram_addr_t ram_addr);
@@ -1410,14 +1467,18 @@ static void breakpoint_invalidate(CPUState *env, target_ulong pc)
static void breakpoint_invalidate(CPUState *env, target_ulong pc)
{
target_phys_addr_t addr;
- target_ulong pd;
ram_addr_t ram_addr;
- PhysPageDesc p;
+ MemoryRegionSection section;
addr = cpu_get_phys_page_debug(env, pc);
- p = phys_page_find(addr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
- ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
+ section = phys_page_find(addr >> TARGET_PAGE_BITS);
+ if (!(memory_region_is_ram(section.mr)
+ || (section.mr->rom_device && section.mr->readable))) {
+ return;
+ }
+ ram_addr = (memory_region_get_ram_addr(section.mr)
+ + section.offset_within_region) & TARGET_PAGE_MASK;
+ ram_addr |= (pc & ~TARGET_PAGE_MASK);
tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
}
#endif
@@ -2095,24 +2156,21 @@ static void tlb_add_large_page(CPUState *env, target_ulong vaddr,
env->tlb_flush_mask = mask;
}
-static bool is_ram_rom(ram_addr_t pd)
+static bool is_ram_rom(MemoryRegionSection *s)
{
- pd &= ~TARGET_PAGE_MASK;
- return pd == io_mem_ram.ram_addr || pd == io_mem_rom.ram_addr;
+ return memory_region_is_ram(s->mr);
}
-static bool is_romd(ram_addr_t pd)
+static bool is_romd(MemoryRegionSection *s)
{
- MemoryRegion *mr;
+ MemoryRegion *mr = s->mr;
- pd &= ~TARGET_PAGE_MASK;
- mr = io_mem_region[pd];
return mr->rom_device && mr->readable;
}
-static bool is_ram_rom_romd(ram_addr_t pd)
+static bool is_ram_rom_romd(MemoryRegionSection *s)
{
- return is_ram_rom(pd) || is_romd(pd);
+ return is_ram_rom(s) || is_romd(s);
}
/* Add a new TLB entry. At most one entry for a given virtual address
@@ -2122,8 +2180,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
target_phys_addr_t paddr, int prot,
int mmu_idx, target_ulong size)
{
- PhysPageDesc p;
- unsigned long pd;
+ MemoryRegionSection section;
unsigned int index;
target_ulong address;
target_ulong code_address;
@@ -2136,8 +2193,7 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
if (size != TARGET_PAGE_SIZE) {
tlb_add_large_page(env, vaddr, size);
}
- p = phys_page_find(paddr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(paddr >> TARGET_PAGE_BITS);
#if defined(DEBUG_TLB)
printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
" prot=%x idx=%d pd=0x%08lx\n",
@@ -2145,15 +2201,21 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
#endif
address = vaddr;
- if (!is_ram_rom_romd(pd)) {
+ if (!is_ram_rom_romd(&section)) {
/* IO memory case (romd handled later) */
address |= TLB_MMIO;
}
- addend = (unsigned long)qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
- if (is_ram_rom(pd)) {
+ if (is_ram_rom_romd(&section)) {
+ addend = (unsigned long)(memory_region_get_ram_ptr(section.mr)
+ + section.offset_within_region);
+ } else {
+ addend = 0;
+ }
+ if (is_ram_rom(&section)) {
/* Normal RAM. */
- iotlb = pd & TARGET_PAGE_MASK;
- if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr)
+ iotlb = (memory_region_get_ram_addr(section.mr)
+ + section.offset_within_region) & TARGET_PAGE_MASK;
+ if (!section.readonly)
iotlb |= io_mem_notdirty.ram_addr;
else
iotlb |= io_mem_rom.ram_addr;
@@ -2164,8 +2226,8 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
and avoid full address decoding in every device.
We can't use the high bits of pd for this because
IO_MEM_ROMD uses these as a ram address. */
- iotlb = (pd & ~TARGET_PAGE_MASK);
- iotlb += p.region_offset;
+ iotlb = memory_region_get_ram_addr(section.mr) & ~TARGET_PAGE_MASK;
+ iotlb += section.offset_within_region;
}
code_address = address;
@@ -2198,11 +2260,14 @@ void tlb_set_page(CPUState *env, target_ulong vaddr,
te->addr_code = -1;
}
if (prot & PAGE_WRITE) {
- if ((pd & ~TARGET_PAGE_MASK) == io_mem_rom.ram_addr || is_romd(pd)) {
+ if ((memory_region_is_ram(section.mr) && section.readonly)
+ || is_romd(&section)) {
/* Write access calls the I/O callback. */
te->addr_write = address | TLB_MMIO;
- } else if ((pd & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr &&
- !cpu_physical_memory_is_dirty(pd)) {
+ } else if (memory_region_is_ram(section.mr)
+ && !cpu_physical_memory_is_dirty(
+ section.mr->ram_addr
+ + section.offset_within_region)) {
te->addr_write = address | TLB_NOTDIRTY;
} else {
te->addr_write = address;
@@ -2491,34 +2556,66 @@ static inline void tlb_set_dirty(CPUState *env,
typedef struct subpage_t {
MemoryRegion iomem;
target_phys_addr_t base;
- ram_addr_t sub_io_index[TARGET_PAGE_SIZE];
- ram_addr_t region_offset[TARGET_PAGE_SIZE];
+ uint16_t sub_section[TARGET_PAGE_SIZE];
} subpage_t;
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
- ram_addr_t memory, ram_addr_t region_offset);
-static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
- ram_addr_t orig_memory,
- ram_addr_t region_offset);
-#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
- need_subpage) \
- do { \
- if (addr > start_addr) \
- start_addr2 = 0; \
- else { \
- start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
- if (start_addr2 > 0) \
- need_subpage = 1; \
- } \
- \
- if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
- end_addr2 = TARGET_PAGE_SIZE - 1; \
- else { \
- end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
- if (end_addr2 < TARGET_PAGE_SIZE - 1) \
- need_subpage = 1; \
- } \
- } while (0)
+ uint16_t section);
+static subpage_t *subpage_init(target_phys_addr_t base);
+static void destroy_page_desc(uint16_t section_index)
+{
+ MemoryRegionSection *section = &phys_sections[section_index];
+ MemoryRegion *mr = section->mr;
+
+ if (mr->subpage) {
+ subpage_t *subpage = container_of(mr, subpage_t, iomem);
+ memory_region_destroy(&subpage->iomem);
+ g_free(subpage);
+ }
+}
+
+static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
+{
+ unsigned i;
+ PhysPageEntry *p;
+
+ if (lp->ptr == PHYS_MAP_NODE_NIL) {
+ return;
+ }
+
+ p = phys_map_nodes[lp->ptr];
+ for (i = 0; i < L2_SIZE; ++i) {
+ if (!p[i].is_leaf) {
+ destroy_l2_mapping(&p[i], level - 1);
+ } else {
+ destroy_page_desc(p[i].ptr);
+ }
+ }
+ lp->is_leaf = 0;
+ lp->ptr = PHYS_MAP_NODE_NIL;
+}
+
+static void destroy_all_mappings(void)
+{
+ destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1);
+ phys_map_nodes_reset();
+}
+
+static uint16_t phys_section_add(MemoryRegionSection *section)
+{
+ if (phys_sections_nb == phys_sections_nb_alloc) {
+ phys_sections_nb_alloc = MAX(phys_sections_nb_alloc * 2, 16);
+ phys_sections = g_renew(MemoryRegionSection, phys_sections,
+ phys_sections_nb_alloc);
+ }
+ phys_sections[phys_sections_nb] = *section;
+ return phys_sections_nb++;
+}
+
+static void phys_sections_clear(void)
+{
+ phys_sections_nb = 0;
+}
/* register physical memory.
For RAM, 'size' must be a multiple of the target page size.
@@ -2528,101 +2625,78 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
start_addr and region_offset are rounded down to a page boundary
before calculating this offset. This should not be a problem unless
the low bits of start_addr and region_offset differ. */
-void cpu_register_physical_memory_log(MemoryRegionSection *section,
- bool readable, bool readonly)
+static void register_subpage(MemoryRegionSection *section)
{
- target_phys_addr_t start_addr = section->offset_within_address_space;
- ram_addr_t size = section->size;
- ram_addr_t phys_offset = section->mr->ram_addr;
- ram_addr_t region_offset = section->offset_within_region;
- target_phys_addr_t addr, end_addr;
- PhysPageDesc *p;
- CPUState *env;
- ram_addr_t orig_size = size;
subpage_t *subpage;
-
- if (memory_region_is_ram(section->mr)) {
- phys_offset += region_offset;
- region_offset = 0;
+ target_phys_addr_t base = section->offset_within_address_space
+ & TARGET_PAGE_MASK;
+ MemoryRegionSection existing = phys_page_find(base >> TARGET_PAGE_BITS);
+ MemoryRegionSection subsection = {
+ .offset_within_address_space = base,
+ .size = TARGET_PAGE_SIZE,
+ };
+ target_phys_addr_t start, end;
+
+ assert(existing.mr->subpage || existing.mr == &io_mem_unassigned);
+
+ if (!(existing.mr->subpage)) {
+ subpage = subpage_init(base);
+ subsection.mr = &subpage->iomem;
+ phys_page_set(base >> TARGET_PAGE_BITS, 1,
+ phys_section_add(&subsection));
+ } else {
+ subpage = container_of(existing.mr, subpage_t, iomem);
}
+ start = section->offset_within_address_space & ~TARGET_PAGE_MASK;
+ end = start + section->size;
+ subpage_register(subpage, start, end, phys_section_add(section));
+}
- if (readonly) {
- phys_offset |= io_mem_rom.ram_addr;
- }
- assert(size);
+static void register_multipage(MemoryRegionSection *section)
+{
+ target_phys_addr_t start_addr = section->offset_within_address_space;
+ ram_addr_t size = section->size;
+ target_phys_addr_t addr;
+ uint16_t section_index = phys_section_add(section);
- if (phys_offset == io_mem_unassigned.ram_addr) {
- region_offset = start_addr;
- }
- region_offset &= TARGET_PAGE_MASK;
- size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
- end_addr = start_addr + (target_phys_addr_t)size;
+ assert(size);
addr = start_addr;
- do {
- p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 0);
- if (p && p->phys_offset != io_mem_unassigned.ram_addr) {
- ram_addr_t orig_memory = p->phys_offset;
- target_phys_addr_t start_addr2, end_addr2;
- int need_subpage = 0;
- MemoryRegion *mr = io_mem_region[orig_memory & ~TARGET_PAGE_MASK];
-
- CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
- need_subpage);
- if (need_subpage) {
- if (!(mr->subpage)) {
- subpage = subpage_init((addr & TARGET_PAGE_MASK),
- &p->phys_offset, orig_memory,
- p->region_offset);
- } else {
- subpage = container_of(mr, subpage_t, iomem);
- }
- subpage_register(subpage, start_addr2, end_addr2, phys_offset,
- region_offset);
- p->region_offset = 0;
- } else {
- p->phys_offset = phys_offset;
- p->region_offset = region_offset;
- if (is_ram_rom_romd(phys_offset))
- phys_offset += TARGET_PAGE_SIZE;
- }
- } else {
- p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
- p->phys_offset = phys_offset;
- p->region_offset = region_offset;
- if (is_ram_rom_romd(phys_offset)) {
- phys_offset += TARGET_PAGE_SIZE;
- } else {
- target_phys_addr_t start_addr2, end_addr2;
- int need_subpage = 0;
-
- CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
- end_addr2, need_subpage);
-
- if (need_subpage) {
- subpage = subpage_init((addr & TARGET_PAGE_MASK),
- &p->phys_offset,
- io_mem_unassigned.ram_addr,
- addr & TARGET_PAGE_MASK);
- subpage_register(subpage, start_addr2, end_addr2,
- phys_offset, region_offset);
- p->region_offset = 0;
- }
- }
- }
- region_offset += TARGET_PAGE_SIZE;
- addr += TARGET_PAGE_SIZE;
- } while (addr != end_addr);
+ phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
+ section_index);
+}
- /* since each CPU stores ram addresses in its TLB cache, we must
- reset the modified entries */
- /* XXX: slow ! */
- for(env = first_cpu; env != NULL; env = env->next_cpu) {
- tlb_flush(env, 1);
+void cpu_register_physical_memory_log(MemoryRegionSection *section,
+ bool readonly)
+{
+ MemoryRegionSection now = *section, remain = *section;
+
+ if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
+ || (now.size < TARGET_PAGE_SIZE)) {
+ now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
+ - now.offset_within_address_space,
+ now.size);
+ register_subpage(&now);
+ remain.size -= now.size;
+ remain.offset_within_address_space += now.size;
+ remain.offset_within_region += now.size;
+ }
+ now = remain;
+ now.size &= TARGET_PAGE_MASK;
+ if (now.size) {
+ register_multipage(&now);
+ remain.size -= now.size;
+ remain.offset_within_address_space += now.size;
+ remain.offset_within_region += now.size;
+ }
+ now = remain;
+ if (now.size) {
+ register_subpage(&now);
}
}
+
void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
{
if (kvm_enabled())
@@ -3307,14 +3381,17 @@ static uint64_t subpage_read(void *opaque, target_phys_addr_t addr,
{
subpage_t *mmio = opaque;
unsigned int idx = SUBPAGE_IDX(addr);
+ MemoryRegionSection *section;
#if defined(DEBUG_SUBPAGE)
printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
mmio, len, addr, idx);
#endif
- addr += mmio->region_offset[idx];
- idx = mmio->sub_io_index[idx];
- return io_mem_read(idx, addr, len);
+ section = &phys_sections[mmio->sub_section[idx]];
+ addr += mmio->base;
+ addr -= section->offset_within_address_space;
+ addr += section->offset_within_region;
+ return io_mem_read(section->mr->ram_addr, addr, len);
}
static void subpage_write(void *opaque, target_phys_addr_t addr,
@@ -3322,15 +3399,18 @@ static void subpage_write(void *opaque, target_phys_addr_t addr,
{
subpage_t *mmio = opaque;
unsigned int idx = SUBPAGE_IDX(addr);
+ MemoryRegionSection *section;
#if defined(DEBUG_SUBPAGE)
printf("%s: subpage %p len %d addr " TARGET_FMT_plx
" idx %d value %"PRIx64"\n",
__func__, mmio, len, addr, idx, value);
#endif
- addr += mmio->region_offset[idx];
- idx = mmio->sub_io_index[idx];
- io_mem_write(idx, addr, value, len);
+ section = &phys_sections[mmio->sub_section[idx]];
+ addr += mmio->base;
+ addr -= section->offset_within_address_space;
+ addr += section->offset_within_region;
+ io_mem_write(section->mr->ram_addr, addr, value, len);
}
static const MemoryRegionOps subpage_ops = {
@@ -3372,7 +3452,7 @@ static const MemoryRegionOps subpage_ram_ops = {
};
static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
- ram_addr_t memory, ram_addr_t region_offset)
+ uint16_t section)
{
int idx, eidx;
@@ -3384,24 +3464,21 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %ld\n", __func__,
mmio, start, end, idx, eidx, memory);
#endif
- if ((memory & ~TARGET_PAGE_MASK) == io_mem_ram.ram_addr) {
- memory = io_mem_subpage_ram.ram_addr;
+ if (memory_region_is_ram(phys_sections[section].mr)) {
+ MemoryRegionSection new_section = phys_sections[section];
+ new_section.mr = &io_mem_subpage_ram;
+ section = phys_section_add(&new_section);
}
- memory &= IO_MEM_NB_ENTRIES - 1;
for (; idx <= eidx; idx++) {
- mmio->sub_io_index[idx] = memory;
- mmio->region_offset[idx] = region_offset;
+ mmio->sub_section[idx] = section;
}
return 0;
}
-static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
- ram_addr_t orig_memory,
- ram_addr_t region_offset)
+static subpage_t *subpage_init(target_phys_addr_t base)
{
subpage_t *mmio;
- int subpage_memory;
mmio = g_malloc0(sizeof(subpage_t));
@@ -3409,13 +3486,11 @@ static subpage_t *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
memory_region_init_io(&mmio->iomem, &subpage_ops, mmio,
"subpage", TARGET_PAGE_SIZE);
mmio->iomem.subpage = true;
- subpage_memory = mmio->iomem.ram_addr;
#if defined(DEBUG_SUBPAGE)
printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
mmio, base, TARGET_PAGE_SIZE, subpage_memory);
#endif
- *phys = subpage_memory;
- subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, orig_memory, region_offset);
+ subpage_register(mmio, 0, TARGET_PAGE_SIZE-1, phys_section_unassigned);
return mmio;
}
@@ -3467,6 +3542,18 @@ void cpu_unregister_io_memory(int io_index)
io_mem_used[io_index] = 0;
}
+static uint16_t dummy_section(MemoryRegion *mr)
+{
+ MemoryRegionSection section = {
+ .mr = mr,
+ .offset_within_address_space = 0,
+ .offset_within_region = 0,
+ .size = UINT64_MAX,
+ };
+
+ return phys_section_add(&section);
+}
+
static void io_mem_init(void)
{
int i;
@@ -3488,6 +3575,174 @@ static void io_mem_init(void)
"watch", UINT64_MAX);
}
+static void core_begin(MemoryListener *listener)
+{
+ destroy_all_mappings();
+ phys_sections_clear();
+ phys_map.ptr = PHYS_MAP_NODE_NIL;
+ phys_section_unassigned = dummy_section(&io_mem_unassigned);
+}
+
+static void core_commit(MemoryListener *listener)
+{
+ CPUState *env;
+
+ /* since each CPU stores ram addresses in its TLB cache, we must
+ reset the modified entries */
+ /* XXX: slow ! */
+ for(env = first_cpu; env != NULL; env = env->next_cpu) {
+ tlb_flush(env, 1);
+ }
+}
+
+static void core_region_add(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+ cpu_register_physical_memory_log(section, section->readonly);
+}
+
+static void core_region_del(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
+static void core_region_nop(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+ cpu_register_physical_memory_log(section, section->readonly);
+}
+
+static void core_log_start(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
+static void core_log_stop(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
+static void core_log_sync(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
+static void core_log_global_start(MemoryListener *listener)
+{
+ cpu_physical_memory_set_dirty_tracking(1);
+}
+
+static void core_log_global_stop(MemoryListener *listener)
+{
+ cpu_physical_memory_set_dirty_tracking(0);
+}
+
+static void core_eventfd_add(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+}
+
+static void core_eventfd_del(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+}
+
+static void io_begin(MemoryListener *listener)
+{
+}
+
+static void io_commit(MemoryListener *listener)
+{
+}
+
+static void io_region_add(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+ iorange_init(&section->mr->iorange, &memory_region_iorange_ops,
+ section->offset_within_address_space, section->size);
+ ioport_register(&section->mr->iorange);
+}
+
+static void io_region_del(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+ isa_unassign_ioport(section->offset_within_address_space, section->size);
+}
+
+static void io_region_nop(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
+static void io_log_start(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
+static void io_log_stop(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
+static void io_log_sync(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
+static void io_log_global_start(MemoryListener *listener)
+{
+}
+
+static void io_log_global_stop(MemoryListener *listener)
+{
+}
+
+static void io_eventfd_add(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+}
+
+static void io_eventfd_del(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+}
+
+static MemoryListener core_memory_listener = {
+ .begin = core_begin,
+ .commit = core_commit,
+ .region_add = core_region_add,
+ .region_del = core_region_del,
+ .region_nop = core_region_nop,
+ .log_start = core_log_start,
+ .log_stop = core_log_stop,
+ .log_sync = core_log_sync,
+ .log_global_start = core_log_global_start,
+ .log_global_stop = core_log_global_stop,
+ .eventfd_add = core_eventfd_add,
+ .eventfd_del = core_eventfd_del,
+ .priority = 0,
+};
+
+static MemoryListener io_memory_listener = {
+ .begin = io_begin,
+ .commit = io_commit,
+ .region_add = io_region_add,
+ .region_del = io_region_del,
+ .region_nop = io_region_nop,
+ .log_start = io_log_start,
+ .log_stop = io_log_stop,
+ .log_sync = io_log_sync,
+ .log_global_start = io_log_global_start,
+ .log_global_stop = io_log_global_stop,
+ .eventfd_add = io_eventfd_add,
+ .eventfd_del = io_eventfd_del,
+ .priority = 0,
+};
+
static void memory_map_init(void)
{
system_memory = g_malloc(sizeof(*system_memory));
@@ -3497,6 +3752,9 @@ static void memory_map_init(void)
system_io = g_malloc(sizeof(*system_io));
memory_region_init(system_io, "io", 65536);
set_system_io_map(system_io);
+
+ memory_listener_register(&core_memory_listener, system_memory);
+ memory_listener_register(&io_memory_listener, system_io);
}
MemoryRegion *get_system_memory(void)
@@ -3560,22 +3818,22 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
uint8_t *ptr;
uint32_t val;
target_phys_addr_t page;
- ram_addr_t pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
while (len > 0) {
page = addr & TARGET_PAGE_MASK;
l = (page + TARGET_PAGE_SIZE) - addr;
if (l > len)
l = len;
- p = phys_page_find(page >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(page >> TARGET_PAGE_BITS);
if (is_write) {
- if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
+ if (!memory_region_is_ram(section.mr)) {
target_phys_addr_t addr1;
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ io_index = memory_region_get_ram_addr(section.mr)
+ & (IO_MEM_NB_ENTRIES - 1);
+ addr1 = (addr & ~TARGET_PAGE_MASK)
+ + section.offset_within_region;
/* XXX: could force cpu_single_env to NULL to avoid
potential bugs */
if (l >= 4 && ((addr1 & 3) == 0)) {
@@ -3594,9 +3852,11 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
io_mem_write(io_index, addr1, val, 1);
l = 1;
}
- } else {
+ } else if (!section.readonly) {
ram_addr_t addr1;
- addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+ addr1 = (memory_region_get_ram_addr(section.mr)
+ + section.offset_within_region)
+ | (addr & ~TARGET_PAGE_MASK);
/* RAM case */
ptr = qemu_get_ram_ptr(addr1);
memcpy(ptr, buf, l);
@@ -3610,11 +3870,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
qemu_put_ram_ptr(ptr);
}
} else {
- if (!is_ram_rom_romd(pd)) {
+ if (!is_ram_rom_romd(&section)) {
target_phys_addr_t addr1;
/* I/O case */
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr1 = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ io_index = memory_region_get_ram_addr(section.mr)
+ & (IO_MEM_NB_ENTRIES - 1);
+ addr1 = (addr & ~TARGET_PAGE_MASK)
+ + section.offset_within_region;
if (l >= 4 && ((addr1 & 3) == 0)) {
/* 32 bit read access */
val = io_mem_read(io_index, addr1, 4);
@@ -3633,7 +3895,8 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
}
} else {
/* RAM case */
- ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK);
+ ptr = qemu_get_ram_ptr(section.mr->ram_addr
+ + section.offset_within_region);
memcpy(buf, ptr + (addr & ~TARGET_PAGE_MASK), l);
qemu_put_ram_ptr(ptr);
}
@@ -3651,22 +3914,22 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
int l;
uint8_t *ptr;
target_phys_addr_t page;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
while (len > 0) {
page = addr & TARGET_PAGE_MASK;
l = (page + TARGET_PAGE_SIZE) - addr;
if (l > len)
l = len;
- p = phys_page_find(page >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(page >> TARGET_PAGE_BITS);
- if (!is_ram_rom_romd(pd)) {
+ if (!is_ram_rom_romd(&section)) {
/* do nothing */
} else {
unsigned long addr1;
- addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+ addr1 = (memory_region_get_ram_addr(section.mr)
+ + section.offset_within_region)
+ + (addr & ~TARGET_PAGE_MASK);
/* ROM/RAM case */
ptr = qemu_get_ram_ptr(addr1);
memcpy(ptr, buf, l);
@@ -3739,8 +4002,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
target_phys_addr_t todo = 0;
int l;
target_phys_addr_t page;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
ram_addr_t raddr = RAM_ADDR_MAX;
ram_addr_t rlen;
void *ret;
@@ -3750,10 +4012,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
l = (page + TARGET_PAGE_SIZE) - addr;
if (l > len)
l = len;
- p = phys_page_find(page >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(page >> TARGET_PAGE_BITS);
- if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
+ if (!(memory_region_is_ram(section.mr) && !section.readonly)) {
if (todo || bounce.buffer) {
break;
}
@@ -3768,7 +4029,9 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
return bounce.buffer;
}
if (!todo) {
- raddr = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+ raddr = memory_region_get_ram_addr(section.mr)
+ + section.offset_within_region
+ + (addr & ~TARGET_PAGE_MASK);
}
len -= l;
@@ -3827,16 +4090,15 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
int io_index;
uint8_t *ptr;
uint32_t val;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
- p = phys_page_find(addr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(addr >> TARGET_PAGE_BITS);
- if (!is_ram_rom_romd(pd)) {
+ if (!is_ram_rom_romd(&section)) {
/* I/O case */
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ io_index = memory_region_get_ram_addr(section.mr)
+ & (IO_MEM_NB_ENTRIES - 1);
+ addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
val = io_mem_read(io_index, addr, 4);
#if defined(TARGET_WORDS_BIGENDIAN)
if (endian == DEVICE_LITTLE_ENDIAN) {
@@ -3849,7 +4111,9 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
#endif
} else {
/* RAM case */
- ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
+ ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr)
+ & TARGET_PAGE_MASK)
+ + section.offset_within_region) +
(addr & ~TARGET_PAGE_MASK);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
@@ -3888,16 +4152,15 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
int io_index;
uint8_t *ptr;
uint64_t val;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
- p = phys_page_find(addr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(addr >> TARGET_PAGE_BITS);
- if (!is_ram_rom_romd(pd)) {
+ if (!is_ram_rom_romd(&section)) {
/* I/O case */
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ io_index = memory_region_get_ram_addr(section.mr)
+ & (IO_MEM_NB_ENTRIES - 1);
+ addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
/* XXX This is broken when device endian != cpu endian.
Fix and add "endian" variable check */
@@ -3910,8 +4173,10 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
#endif
} else {
/* RAM case */
- ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
- (addr & ~TARGET_PAGE_MASK);
+ ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr)
+ & TARGET_PAGE_MASK)
+ + section.offset_within_region)
+ + (addr & ~TARGET_PAGE_MASK);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
val = ldq_le_p(ptr);
@@ -3957,16 +4222,15 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
int io_index;
uint8_t *ptr;
uint64_t val;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
- p = phys_page_find(addr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(addr >> TARGET_PAGE_BITS);
- if (!is_ram_rom_romd(pd)) {
+ if (!is_ram_rom_romd(&section)) {
/* I/O case */
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ io_index = memory_region_get_ram_addr(section.mr)
+ & (IO_MEM_NB_ENTRIES - 1);
+ addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
val = io_mem_read(io_index, addr, 2);
#if defined(TARGET_WORDS_BIGENDIAN)
if (endian == DEVICE_LITTLE_ENDIAN) {
@@ -3979,8 +4243,10 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
#endif
} else {
/* RAM case */
- ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
- (addr & ~TARGET_PAGE_MASK);
+ ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr)
+ & TARGET_PAGE_MASK)
+ + section.offset_within_region)
+ + (addr & ~TARGET_PAGE_MASK);
switch (endian) {
case DEVICE_LITTLE_ENDIAN:
val = lduw_le_p(ptr);
@@ -4018,18 +4284,23 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
{
int io_index;
uint8_t *ptr;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
- p = phys_page_find(addr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(addr >> TARGET_PAGE_BITS);
- if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ if (!memory_region_is_ram(section.mr) || section.readonly) {
+ if (memory_region_is_ram(section.mr)) {
+ io_index = io_mem_rom.ram_addr;
+ } else {
+ io_index = memory_region_get_ram_addr(section.mr);
+ }
+ addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
io_mem_write(io_index, addr, val, 4);
} else {
- unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+ unsigned long addr1 = (memory_region_get_ram_addr(section.mr)
+ & TARGET_PAGE_MASK)
+ + section.offset_within_region
+ + (addr & ~TARGET_PAGE_MASK);
ptr = qemu_get_ram_ptr(addr1);
stl_p(ptr, val);
@@ -4049,15 +4320,18 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
{
int io_index;
uint8_t *ptr;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
- p = phys_page_find(addr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(addr >> TARGET_PAGE_BITS);
- if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ if (!memory_region_is_ram(section.mr) || section.readonly) {
+ if (memory_region_is_ram(section.mr)) {
+ io_index = io_mem_rom.ram_addr;
+ } else {
+ io_index = memory_region_get_ram_addr(section.mr)
+ & (IO_MEM_NB_ENTRIES - 1);
+ }
+ addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
#ifdef TARGET_WORDS_BIGENDIAN
io_mem_write(io_index, addr, val >> 32, 4);
io_mem_write(io_index, addr + 4, (uint32_t)val, 4);
@@ -4066,8 +4340,10 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
io_mem_write(io_index, addr + 4, val >> 32, 4);
#endif
} else {
- ptr = qemu_get_ram_ptr(pd & TARGET_PAGE_MASK) +
- (addr & ~TARGET_PAGE_MASK);
+ ptr = qemu_get_ram_ptr((memory_region_get_ram_addr(section.mr)
+ & TARGET_PAGE_MASK)
+ + section.offset_within_region)
+ + (addr & ~TARGET_PAGE_MASK);
stq_p(ptr, val);
}
}
@@ -4078,15 +4354,18 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
{
int io_index;
uint8_t *ptr;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
- p = phys_page_find(addr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(addr >> TARGET_PAGE_BITS);
- if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ if (!memory_region_is_ram(section.mr) || section.readonly) {
+ if (memory_region_is_ram(section.mr)) {
+ io_index = io_mem_rom.ram_addr;
+ } else {
+ io_index = memory_region_get_ram_addr(section.mr)
+ & (IO_MEM_NB_ENTRIES - 1);
+ }
+ addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
#if defined(TARGET_WORDS_BIGENDIAN)
if (endian == DEVICE_LITTLE_ENDIAN) {
val = bswap32(val);
@@ -4099,7 +4378,9 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
io_mem_write(io_index, addr, val, 4);
} else {
unsigned long addr1;
- addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+ addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK)
+ + section.offset_within_region
+ + (addr & ~TARGET_PAGE_MASK);
/* RAM case */
ptr = qemu_get_ram_ptr(addr1);
switch (endian) {
@@ -4151,15 +4432,18 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
{
int io_index;
uint8_t *ptr;
- unsigned long pd;
- PhysPageDesc p;
+ MemoryRegionSection section;
- p = phys_page_find(addr >> TARGET_PAGE_BITS);
- pd = p.phys_offset;
+ section = phys_page_find(addr >> TARGET_PAGE_BITS);
- if ((pd & ~TARGET_PAGE_MASK) != io_mem_ram.ram_addr) {
- io_index = pd & (IO_MEM_NB_ENTRIES - 1);
- addr = (addr & ~TARGET_PAGE_MASK) + p.region_offset;
+ if (!memory_region_is_ram(section.mr) || section.readonly) {
+ if (memory_region_is_ram(section.mr)) {
+ io_index = io_mem_rom.ram_addr;
+ } else {
+ io_index = memory_region_get_ram_addr(section.mr)
+ & (IO_MEM_NB_ENTRIES - 1);
+ }
+ addr = (addr & ~TARGET_PAGE_MASK) + section.offset_within_region;
#if defined(TARGET_WORDS_BIGENDIAN)
if (endian == DEVICE_LITTLE_ENDIAN) {
val = bswap16(val);
@@ -4172,7 +4456,8 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
io_mem_write(io_index, addr, val, 2);
} else {
unsigned long addr1;
- addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
+ addr1 = (memory_region_get_ram_addr(section.mr) & TARGET_PAGE_MASK)
+ + section.offset_within_region + (addr & ~TARGET_PAGE_MASK);
/* RAM case */
ptr = qemu_get_ram_ptr(addr1);
switch (endian) {
@@ -4389,7 +4674,7 @@ tb_page_addr_t get_page_addr_code(CPUState *env1, target_ulong addr)
}
pd = env1->tlb_table[mmu_idx][page_index].addr_code & ~TARGET_PAGE_MASK;
if (pd != io_mem_ram.ram_addr && pd != io_mem_rom.ram_addr
- && !is_romd(pd)) {
+ && !io_mem_region[pd]->rom_device) {
#if defined(TARGET_ALPHA) || defined(TARGET_MIPS) || defined(TARGET_SPARC)
cpu_unassigned_access(env1, addr, 0, 1, 0, 4);
#else
diff --git a/hw/vhost.c b/hw/vhost.c
index 5ece659f4a..8d3ba5b608 100644
--- a/hw/vhost.c
+++ b/hw/vhost.c
@@ -436,6 +436,14 @@ static bool vhost_section(MemoryRegionSection *section)
&& memory_region_is_ram(section->mr);
}
+static void vhost_begin(MemoryListener *listener)
+{
+}
+
+static void vhost_commit(MemoryListener *listener)
+{
+}
+
static void vhost_region_add(MemoryListener *listener,
MemoryRegionSection *section)
{
@@ -476,6 +484,11 @@ static void vhost_region_del(MemoryListener *listener,
}
}
+static void vhost_region_nop(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
struct vhost_virtqueue *vq,
unsigned idx, bool enable_log)
@@ -720,6 +733,18 @@ static void vhost_virtqueue_cleanup(struct vhost_dev *dev,
0, virtio_queue_get_desc_size(vdev, idx));
}
+static void vhost_eventfd_add(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+}
+
+static void vhost_eventfd_del(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+}
+
int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
{
uint64_t features;
@@ -744,13 +769,19 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
hdev->features = features;
hdev->memory_listener = (MemoryListener) {
+ .begin = vhost_begin,
+ .commit = vhost_commit,
.region_add = vhost_region_add,
.region_del = vhost_region_del,
+ .region_nop = vhost_region_nop,
.log_start = vhost_log_start,
.log_stop = vhost_log_stop,
.log_sync = vhost_log_sync,
.log_global_start = vhost_log_global_start,
.log_global_stop = vhost_log_global_stop,
+ .eventfd_add = vhost_eventfd_add,
+ .eventfd_del = vhost_eventfd_del,
+ .priority = 10
};
hdev->mem = g_malloc0(offsetof(struct vhost_memory, regions));
hdev->n_mem_sections = 0;
@@ -759,7 +790,7 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
hdev->log_size = 0;
hdev->log_enabled = false;
hdev->started = false;
- memory_listener_register(&hdev->memory_listener);
+ memory_listener_register(&hdev->memory_listener, NULL);
hdev->force = force;
return 0;
fail:
diff --git a/ioport.c b/ioport.c
index 36fa3a477e..8a474d3492 100644
--- a/ioport.c
+++ b/ioport.c
@@ -328,6 +328,7 @@ void portio_list_init(PortioList *piolist,
piolist->ports = callbacks;
piolist->nr = 0;
piolist->regions = g_new0(MemoryRegion *, n);
+ piolist->aliases = g_new0(MemoryRegion *, n);
piolist->address_space = NULL;
piolist->opaque = opaque;
piolist->name = name;
@@ -336,6 +337,7 @@ void portio_list_init(PortioList *piolist,
void portio_list_destroy(PortioList *piolist)
{
g_free(piolist->regions);
+ g_free(piolist->aliases);
}
static void portio_list_add_1(PortioList *piolist,
@@ -345,7 +347,7 @@ static void portio_list_add_1(PortioList *piolist,
{
MemoryRegionPortio *pio;
MemoryRegionOps *ops;
- MemoryRegion *region;
+ MemoryRegion *region, *alias;
unsigned i;
/* Copy the sub-list and null-terminate it. */
@@ -362,12 +364,20 @@ static void portio_list_add_1(PortioList *piolist,
ops->old_portio = pio;
region = g_new(MemoryRegion, 1);
+ alias = g_new(MemoryRegion, 1);
+ /*
+ * Use an alias so that the callback is called with an absolute address,
+ * rather than an offset relative to to start + off_low.
+ */
memory_region_init_io(region, ops, piolist->opaque, piolist->name,
- off_high - off_low);
- memory_region_set_offset(region, start + off_low);
+ UINT64_MAX);
+ memory_region_init_alias(alias, piolist->name,
+ region, start + off_low, off_high - off_low);
memory_region_add_subregion(piolist->address_space,
- start + off_low, region);
- piolist->regions[piolist->nr++] = region;
+ start + off_low, alias);
+ piolist->regions[piolist->nr] = region;
+ piolist->aliases[piolist->nr] = alias;
+ ++piolist->nr;
}
void portio_list_add(PortioList *piolist,
@@ -409,15 +419,19 @@ void portio_list_add(PortioList *piolist,
void portio_list_del(PortioList *piolist)
{
- MemoryRegion *mr;
+ MemoryRegion *mr, *alias;
unsigned i;
for (i = 0; i < piolist->nr; ++i) {
mr = piolist->regions[i];
- memory_region_del_subregion(piolist->address_space, mr);
+ alias = piolist->aliases[i];
+ memory_region_del_subregion(piolist->address_space, alias);
+ memory_region_destroy(alias);
memory_region_destroy(mr);
g_free((MemoryRegionOps *)mr->ops);
g_free(mr);
+ g_free(alias);
piolist->regions[i] = NULL;
+ piolist->aliases[i] = NULL;
}
}
diff --git a/ioport.h b/ioport.h
index ae3e9da0b5..ab29c89fb3 100644
--- a/ioport.h
+++ b/ioport.h
@@ -60,6 +60,7 @@ typedef struct PortioList {
struct MemoryRegion *address_space;
unsigned nr;
struct MemoryRegion **regions;
+ struct MemoryRegion **aliases;
void *opaque;
const char *name;
} PortioList;
diff --git a/kvm-all.c b/kvm-all.c
index e2cbc03022..7119751956 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -28,6 +28,7 @@
#include "kvm.h"
#include "bswap.h"
#include "memory.h"
+#include "exec-memory.h"
/* This check must be after config-host.h is included */
#ifdef CONFIG_EVENTFD
@@ -674,6 +675,14 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
}
}
+static void kvm_begin(MemoryListener *listener)
+{
+}
+
+static void kvm_commit(MemoryListener *listener)
+{
+}
+
static void kvm_region_add(MemoryListener *listener,
MemoryRegionSection *section)
{
@@ -686,6 +695,11 @@ static void kvm_region_del(MemoryListener *listener,
kvm_set_phys_mem(section, false);
}
+static void kvm_region_nop(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
static void kvm_log_sync(MemoryListener *listener,
MemoryRegionSection *section)
{
@@ -713,14 +727,95 @@ static void kvm_log_global_stop(struct MemoryListener *listener)
assert(r >= 0);
}
+static void kvm_mem_ioeventfd_add(MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+ int r;
+
+ assert(match_data && section->size == 4);
+
+ r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
+ data, true);
+ if (r < 0) {
+ abort();
+ }
+}
+
+static void kvm_mem_ioeventfd_del(MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+ int r;
+
+ r = kvm_set_ioeventfd_mmio_long(fd, section->offset_within_address_space,
+ data, false);
+ if (r < 0) {
+ abort();
+ }
+}
+
+static void kvm_io_ioeventfd_add(MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+ int r;
+
+ assert(match_data && section->size == 2);
+
+ r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
+ data, true);
+ if (r < 0) {
+ abort();
+ }
+}
+
+static void kvm_io_ioeventfd_del(MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+
+{
+ int r;
+
+ r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
+ data, false);
+ if (r < 0) {
+ abort();
+ }
+}
+
+static void kvm_eventfd_add(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+ if (section->address_space == get_system_memory()) {
+ kvm_mem_ioeventfd_add(section, match_data, data, fd);
+ } else {
+ kvm_io_ioeventfd_add(section, match_data, data, fd);
+ }
+}
+
+static void kvm_eventfd_del(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+ if (section->address_space == get_system_memory()) {
+ kvm_mem_ioeventfd_del(section, match_data, data, fd);
+ } else {
+ kvm_io_ioeventfd_del(section, match_data, data, fd);
+ }
+}
+
static MemoryListener kvm_memory_listener = {
+ .begin = kvm_begin,
+ .commit = kvm_commit,
.region_add = kvm_region_add,
.region_del = kvm_region_del,
+ .region_nop = kvm_region_nop,
.log_start = kvm_log_start,
.log_stop = kvm_log_stop,
.log_sync = kvm_log_sync,
.log_global_start = kvm_log_global_start,
.log_global_stop = kvm_log_global_stop,
+ .eventfd_add = kvm_eventfd_add,
+ .eventfd_del = kvm_eventfd_del,
+ .priority = 10,
};
static void kvm_handle_interrupt(CPUState *env, int mask)
@@ -965,7 +1060,7 @@ int kvm_init(void)
}
kvm_state = s;
- memory_listener_register(&kvm_memory_listener);
+ memory_listener_register(&kvm_memory_listener, NULL);
s->many_ioeventfds = kvm_check_many_ioeventfds();
diff --git a/memory.c b/memory.c
index 22816e20f9..6565e2e696 100644
--- a/memory.c
+++ b/memory.c
@@ -27,8 +27,8 @@ unsigned memory_region_transaction_depth = 0;
static bool memory_region_update_pending = false;
static bool global_dirty_log = false;
-static QLIST_HEAD(, MemoryListener) memory_listeners
- = QLIST_HEAD_INITIALIZER(memory_listeners);
+static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners
+ = QTAILQ_HEAD_INITIALIZER(memory_listeners);
typedef struct AddrRange AddrRange;
@@ -82,6 +82,71 @@ static AddrRange addrrange_intersection(AddrRange r1, AddrRange r2)
return addrrange_make(start, int128_sub(end, start));
}
+enum ListenerDirection { Forward, Reverse };
+
+static bool memory_listener_match(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+ return !listener->address_space_filter
+ || listener->address_space_filter == section->address_space;
+}
+
+#define MEMORY_LISTENER_CALL_GLOBAL(_callback, _direction, _args...) \
+ do { \
+ MemoryListener *_listener; \
+ \
+ switch (_direction) { \
+ case Forward: \
+ QTAILQ_FOREACH(_listener, &memory_listeners, link) { \
+ _listener->_callback(_listener, ##_args); \
+ } \
+ break; \
+ case Reverse: \
+ QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \
+ memory_listeners, link) { \
+ _listener->_callback(_listener, ##_args); \
+ } \
+ break; \
+ default: \
+ abort(); \
+ } \
+ } while (0)
+
+#define MEMORY_LISTENER_CALL(_callback, _direction, _section, _args...) \
+ do { \
+ MemoryListener *_listener; \
+ \
+ switch (_direction) { \
+ case Forward: \
+ QTAILQ_FOREACH(_listener, &memory_listeners, link) { \
+ if (memory_listener_match(_listener, _section)) { \
+ _listener->_callback(_listener, _section, ##_args); \
+ } \
+ } \
+ break; \
+ case Reverse: \
+ QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners, \
+ memory_listeners, link) { \
+ if (memory_listener_match(_listener, _section)) { \
+ _listener->_callback(_listener, _section, ##_args); \
+ } \
+ } \
+ break; \
+ default: \
+ abort(); \
+ } \
+ } while (0)
+
+#define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback) \
+ MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) { \
+ .mr = (fr)->mr, \
+ .address_space = (as)->root, \
+ .offset_within_region = (fr)->offset_in_region, \
+ .size = int128_get64((fr)->addr.size), \
+ .offset_within_address_space = int128_get64((fr)->addr.start), \
+ .readonly = (fr)->readonly, \
+ }))
+
struct CoalescedMemoryRange {
AddrRange addr;
QTAILQ_ENTRY(CoalescedMemoryRange) link;
@@ -158,22 +223,12 @@ typedef struct AddressSpaceOps AddressSpaceOps;
/* A system address space - I/O, memory, etc. */
struct AddressSpace {
- const AddressSpaceOps *ops;
MemoryRegion *root;
FlatView current_map;
int ioeventfd_nb;
MemoryRegionIoeventfd *ioeventfds;
};
-struct AddressSpaceOps {
- void (*range_add)(AddressSpace *as, FlatRange *fr);
- void (*range_del)(AddressSpace *as, FlatRange *fr);
- void (*log_start)(AddressSpace *as, FlatRange *fr);
- void (*log_stop)(AddressSpace *as, FlatRange *fr);
- void (*ioeventfd_add)(AddressSpace *as, MemoryRegionIoeventfd *fd);
- void (*ioeventfd_del)(AddressSpace *as, MemoryRegionIoeventfd *fd);
-};
-
#define FOR_EACH_FLAT_RANGE(var, view) \
for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
@@ -305,74 +360,7 @@ static void access_with_adjusted_size(target_phys_addr_t addr,
}
}
-static void as_memory_range_add(AddressSpace *as, FlatRange *fr)
-{
- MemoryRegionSection section = {
- .mr = fr->mr,
- .offset_within_address_space = int128_get64(fr->addr.start),
- .offset_within_region = fr->offset_in_region,
- .size = int128_get64(fr->addr.size),
- };
-
- cpu_register_physical_memory_log(&section, fr->readable, fr->readonly);
-}
-
-static void as_memory_range_del(AddressSpace *as, FlatRange *fr)
-{
- MemoryRegionSection section = {
- .mr = &io_mem_unassigned,
- .offset_within_address_space = int128_get64(fr->addr.start),
- .offset_within_region = int128_get64(fr->addr.start),
- .size = int128_get64(fr->addr.size),
- };
-
- cpu_register_physical_memory_log(&section, true, false);
-}
-
-static void as_memory_log_start(AddressSpace *as, FlatRange *fr)
-{
-}
-
-static void as_memory_log_stop(AddressSpace *as, FlatRange *fr)
-{
-}
-
-static void as_memory_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd)
-{
- int r;
-
- assert(fd->match_data && int128_get64(fd->addr.size) == 4);
-
- r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start),
- fd->data, true);
- if (r < 0) {
- abort();
- }
-}
-
-static void as_memory_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd)
-{
- int r;
-
- r = kvm_set_ioeventfd_mmio_long(fd->fd, int128_get64(fd->addr.start),
- fd->data, false);
- if (r < 0) {
- abort();
- }
-}
-
-static const AddressSpaceOps address_space_ops_memory = {
- .range_add = as_memory_range_add,
- .range_del = as_memory_range_del,
- .log_start = as_memory_log_start,
- .log_stop = as_memory_log_stop,
- .ioeventfd_add = as_memory_ioeventfd_add,
- .ioeventfd_del = as_memory_ioeventfd_del,
-};
-
-static AddressSpace address_space_memory = {
- .ops = &address_space_ops_memory,
-};
+static AddressSpace address_space_memory;
static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset,
unsigned width, bool write)
@@ -401,17 +389,17 @@ static void memory_region_iorange_read(IORange *iorange,
*data = ((uint64_t)1 << (width * 8)) - 1;
if (mrp) {
- *data = mrp->read(mr->opaque, offset + mr->offset);
+ *data = mrp->read(mr->opaque, offset);
} else if (width == 2) {
mrp = find_portio(mr, offset, 1, false);
assert(mrp);
- *data = mrp->read(mr->opaque, offset + mr->offset) |
- (mrp->read(mr->opaque, offset + mr->offset + 1) << 8);
+ *data = mrp->read(mr->opaque, offset) |
+ (mrp->read(mr->opaque, offset + 1) << 8);
}
return;
}
*data = 0;
- access_with_adjusted_size(offset + mr->offset, data, width,
+ access_with_adjusted_size(offset, data, width,
mr->ops->impl.min_access_size,
mr->ops->impl.max_access_size,
memory_region_read_accessor, mr);
@@ -428,73 +416,27 @@ static void memory_region_iorange_write(IORange *iorange,
const MemoryRegionPortio *mrp = find_portio(mr, offset, width, true);
if (mrp) {
- mrp->write(mr->opaque, offset + mr->offset, data);
+ mrp->write(mr->opaque, offset, data);
} else if (width == 2) {
mrp = find_portio(mr, offset, 1, false);
assert(mrp);
- mrp->write(mr->opaque, offset + mr->offset, data & 0xff);
- mrp->write(mr->opaque, offset + mr->offset + 1, data >> 8);
+ mrp->write(mr->opaque, offset, data & 0xff);
+ mrp->write(mr->opaque, offset + 1, data >> 8);
}
return;
}
- access_with_adjusted_size(offset + mr->offset, &data, width,
+ access_with_adjusted_size(offset, &data, width,
mr->ops->impl.min_access_size,
mr->ops->impl.max_access_size,
memory_region_write_accessor, mr);
}
-static const IORangeOps memory_region_iorange_ops = {
+const IORangeOps memory_region_iorange_ops = {
.read = memory_region_iorange_read,
.write = memory_region_iorange_write,
};
-static void as_io_range_add(AddressSpace *as, FlatRange *fr)
-{
- iorange_init(&fr->mr->iorange, &memory_region_iorange_ops,
- int128_get64(fr->addr.start), int128_get64(fr->addr.size));
- ioport_register(&fr->mr->iorange);
-}
-
-static void as_io_range_del(AddressSpace *as, FlatRange *fr)
-{
- isa_unassign_ioport(int128_get64(fr->addr.start),
- int128_get64(fr->addr.size));
-}
-
-static void as_io_ioeventfd_add(AddressSpace *as, MemoryRegionIoeventfd *fd)
-{
- int r;
-
- assert(fd->match_data && int128_get64(fd->addr.size) == 2);
-
- r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start),
- fd->data, true);
- if (r < 0) {
- abort();
- }
-}
-
-static void as_io_ioeventfd_del(AddressSpace *as, MemoryRegionIoeventfd *fd)
-{
- int r;
-
- r = kvm_set_ioeventfd_pio_word(fd->fd, int128_get64(fd->addr.start),
- fd->data, false);
- if (r < 0) {
- abort();
- }
-}
-
-static const AddressSpaceOps address_space_ops_io = {
- .range_add = as_io_range_add,
- .range_del = as_io_range_del,
- .ioeventfd_add = as_io_ioeventfd_add,
- .ioeventfd_del = as_io_ioeventfd_del,
-};
-
-static AddressSpace address_space_io = {
- .ops = &address_space_ops_io,
-};
+static AddressSpace address_space_io;
static AddressSpace *memory_region_to_address_space(MemoryRegion *mr)
{
@@ -621,6 +563,8 @@ static void address_space_add_del_ioeventfds(AddressSpace *as,
unsigned fds_old_nb)
{
unsigned iold, inew;
+ MemoryRegionIoeventfd *fd;
+ MemoryRegionSection section;
/* Generate a symmetric difference of the old and new fd sets, adding
* and deleting as necessary.
@@ -632,13 +576,27 @@ static void address_space_add_del_ioeventfds(AddressSpace *as,
&& (inew == fds_new_nb
|| memory_region_ioeventfd_before(fds_old[iold],
fds_new[inew]))) {
- as->ops->ioeventfd_del(as, &fds_old[iold]);
+ fd = &fds_old[iold];
+ section = (MemoryRegionSection) {
+ .address_space = as->root,
+ .offset_within_address_space = int128_get64(fd->addr.start),
+ .size = int128_get64(fd->addr.size),
+ };
+ MEMORY_LISTENER_CALL(eventfd_del, Forward, &section,
+ fd->match_data, fd->data, fd->fd);
++iold;
} else if (inew < fds_new_nb
&& (iold == fds_old_nb
|| memory_region_ioeventfd_before(fds_new[inew],
fds_old[iold]))) {
- as->ops->ioeventfd_add(as, &fds_new[inew]);
+ fd = &fds_new[inew];
+ section = (MemoryRegionSection) {
+ .address_space = as->root,
+ .offset_within_address_space = int128_get64(fd->addr.start),
+ .size = int128_get64(fd->addr.size),
+ };
+ MEMORY_LISTENER_CALL(eventfd_add, Reverse, &section,
+ fd->match_data, fd->data, fd->fd);
++inew;
} else {
++iold;
@@ -678,32 +636,6 @@ static void address_space_update_ioeventfds(AddressSpace *as)
as->ioeventfd_nb = ioeventfd_nb;
}
-typedef void ListenerCallback(MemoryListener *listener,
- MemoryRegionSection *mrs);
-
-/* Want "void (&MemoryListener::*callback)(const MemoryRegionSection& s)" */
-static void memory_listener_update_region(FlatRange *fr, AddressSpace *as,
- size_t callback_offset)
-{
- MemoryRegionSection section = {
- .mr = fr->mr,
- .address_space = as->root,
- .offset_within_region = fr->offset_in_region,
- .size = int128_get64(fr->addr.size),
- .offset_within_address_space = int128_get64(fr->addr.start),
- };
- MemoryListener *listener;
-
- QLIST_FOREACH(listener, &memory_listeners, link) {
- ListenerCallback *callback
- = *(ListenerCallback **)((void *)listener + callback_offset);
- callback(listener, &section);
- }
-}
-
-#define MEMORY_LISTENER_UPDATE_REGION(fr, as, callback) \
- memory_listener_update_region(fr, as, offsetof(MemoryListener, callback))
-
static void address_space_update_topology_pass(AddressSpace *as,
FlatView old_view,
FlatView new_view,
@@ -736,8 +668,7 @@ static void address_space_update_topology_pass(AddressSpace *as,
/* In old, but (not in new, or in new but attributes changed). */
if (!adding) {
- MEMORY_LISTENER_UPDATE_REGION(frold, as, region_del);
- as->ops->range_del(as, frold);
+ MEMORY_LISTENER_UPDATE_REGION(frold, as, Reverse, region_del);
}
++iold;
@@ -745,12 +676,11 @@ static void address_space_update_topology_pass(AddressSpace *as,
/* In both (logging may have changed) */
if (adding) {
+ MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_nop);
if (frold->dirty_log_mask && !frnew->dirty_log_mask) {
- MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_stop);
- as->ops->log_stop(as, frnew);
+ MEMORY_LISTENER_UPDATE_REGION(frnew, as, Reverse, log_stop);
} else if (frnew->dirty_log_mask && !frold->dirty_log_mask) {
- as->ops->log_start(as, frnew);
- MEMORY_LISTENER_UPDATE_REGION(frnew, as, log_start);
+ MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, log_start);
}
}
@@ -760,8 +690,7 @@ static void address_space_update_topology_pass(AddressSpace *as,
/* In new */
if (adding) {
- as->ops->range_add(as, frnew);
- MEMORY_LISTENER_UPDATE_REGION(frnew, as, region_add);
+ MEMORY_LISTENER_UPDATE_REGION(frnew, as, Forward, region_add);
}
++inew;
@@ -794,6 +723,8 @@ static void memory_region_update_topology(MemoryRegion *mr)
return;
}
+ MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
+
if (address_space_memory.root) {
address_space_update_topology(&address_space_memory);
}
@@ -801,6 +732,8 @@ static void memory_region_update_topology(MemoryRegion *mr)
address_space_update_topology(&address_space_io);
}
+ MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
+
memory_region_update_pending = false;
}
@@ -863,7 +796,6 @@ void memory_region_init(MemoryRegion *mr,
mr->size = int128_2_64();
}
mr->addr = 0;
- mr->offset = 0;
mr->subpage = false;
mr->enabled = true;
mr->terminates = false;
@@ -925,7 +857,7 @@ static uint64_t memory_region_dispatch_read1(MemoryRegion *mr,
}
/* FIXME: support unaligned access */
- access_with_adjusted_size(addr + mr->offset, &data, size,
+ access_with_adjusted_size(addr, &data, size,
mr->ops->impl.min_access_size,
mr->ops->impl.max_access_size,
memory_region_read_accessor, mr);
@@ -979,7 +911,7 @@ static void memory_region_dispatch_write(MemoryRegion *mr,
}
/* FIXME: support unaligned access */
- access_with_adjusted_size(addr + mr->offset, &data, size,
+ access_with_adjusted_size(addr, &data, size,
mr->ops->impl.min_access_size,
mr->ops->impl.max_access_size,
memory_region_write_accessor, mr);
@@ -1122,11 +1054,6 @@ bool memory_region_is_rom(MemoryRegion *mr)
return mr->ram && mr->readonly;
}
-void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset)
-{
- mr->offset = offset;
-}
-
void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
{
uint8_t mask = 1 << client;
@@ -1156,7 +1083,8 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) {
if (fr->mr == mr) {
- MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory, log_sync);
+ MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory,
+ Forward, log_sync);
}
}
}
@@ -1474,6 +1402,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space,
fr->addr.start));
ret.size = int128_get64(range.size);
ret.offset_within_address_space = int128_get64(range.start);
+ ret.readonly = fr->readonly;
return ret;
}
@@ -1483,30 +1412,20 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space)
FlatRange *fr;
FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
- MEMORY_LISTENER_UPDATE_REGION(fr, as, log_sync);
+ MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync);
}
}
void memory_global_dirty_log_start(void)
{
- MemoryListener *listener;
-
- cpu_physical_memory_set_dirty_tracking(1);
global_dirty_log = true;
- QLIST_FOREACH(listener, &memory_listeners, link) {
- listener->log_global_start(listener);
- }
+ MEMORY_LISTENER_CALL_GLOBAL(log_global_start, Forward);
}
void memory_global_dirty_log_stop(void)
{
- MemoryListener *listener;
-
global_dirty_log = false;
- QLIST_FOREACH(listener, &memory_listeners, link) {
- listener->log_global_stop(listener);
- }
- cpu_physical_memory_set_dirty_tracking(0);
+ MEMORY_LISTENER_CALL_GLOBAL(log_global_stop, Reverse);
}
static void listener_add_address_space(MemoryListener *listener,
@@ -1524,21 +1443,36 @@ static void listener_add_address_space(MemoryListener *listener,
.offset_within_region = fr->offset_in_region,
.size = int128_get64(fr->addr.size),
.offset_within_address_space = int128_get64(fr->addr.start),
+ .readonly = fr->readonly,
};
listener->region_add(listener, &section);
}
}
-void memory_listener_register(MemoryListener *listener)
+void memory_listener_register(MemoryListener *listener, MemoryRegion *filter)
{
- QLIST_INSERT_HEAD(&memory_listeners, listener, link);
+ MemoryListener *other = NULL;
+
+ listener->address_space_filter = filter;
+ if (QTAILQ_EMPTY(&memory_listeners)
+ || listener->priority >= QTAILQ_LAST(&memory_listeners,
+ memory_listeners)->priority) {
+ QTAILQ_INSERT_TAIL(&memory_listeners, listener, link);
+ } else {
+ QTAILQ_FOREACH(other, &memory_listeners, link) {
+ if (listener->priority < other->priority) {
+ break;
+ }
+ }
+ QTAILQ_INSERT_BEFORE(other, listener, link);
+ }
listener_add_address_space(listener, &address_space_memory);
listener_add_address_space(listener, &address_space_io);
}
void memory_listener_unregister(MemoryListener *listener)
{
- QLIST_REMOVE(listener, link);
+ QTAILQ_REMOVE(&memory_listeners, listener, link);
}
void set_system_memory_map(MemoryRegion *mr)
diff --git a/memory.h b/memory.h
index 4cf8d2f27a..b7bccd1968 100644
--- a/memory.h
+++ b/memory.h
@@ -115,7 +115,6 @@ struct MemoryRegion {
MemoryRegion *parent;
Int128 size;
target_phys_addr_t addr;
- target_phys_addr_t offset;
void (*destructor)(MemoryRegion *mr);
ram_addr_t ram_addr;
IORange iorange;
@@ -161,6 +160,7 @@ typedef struct MemoryRegionSection MemoryRegionSection;
* @size: the size of the section; will not exceed @mr's boundaries
* @offset_within_address_space: the address of the first byte of the section
* relative to the region's address space
+ * @readonly: writes to this section are ignored
*/
struct MemoryRegionSection {
MemoryRegion *mr;
@@ -168,6 +168,7 @@ struct MemoryRegionSection {
target_phys_addr_t offset_within_region;
uint64_t size;
target_phys_addr_t offset_within_address_space;
+ bool readonly;
};
typedef struct MemoryListener MemoryListener;
@@ -179,14 +180,24 @@ typedef struct MemoryListener MemoryListener;
* Use with memory_listener_register() and memory_listener_unregister().
*/
struct MemoryListener {
+ void (*begin)(MemoryListener *listener);
+ void (*commit)(MemoryListener *listener);
void (*region_add)(MemoryListener *listener, MemoryRegionSection *section);
void (*region_del)(MemoryListener *listener, MemoryRegionSection *section);
+ void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section);
void (*log_start)(MemoryListener *listener, MemoryRegionSection *section);
void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section);
void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section);
void (*log_global_start)(MemoryListener *listener);
void (*log_global_stop)(MemoryListener *listener);
- QLIST_ENTRY(MemoryListener) link;
+ void (*eventfd_add)(MemoryListener *listener, MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd);
+ void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd);
+ /* Lower = earlier (during add), later (during del) */
+ unsigned priority;
+ MemoryRegion *address_space_filter;
+ QTAILQ_ENTRY(MemoryListener) link;
};
/**
@@ -359,14 +370,6 @@ bool memory_region_is_rom(MemoryRegion *mr);
void *memory_region_get_ram_ptr(MemoryRegion *mr);
/**
- * memory_region_set_offset: Sets an offset to be added to MemoryRegionOps
- * callbacks.
- *
- * This function is deprecated and should not be used in new code.
- */
-void memory_region_set_offset(MemoryRegion *mr, target_phys_addr_t offset);
-
-/**
* memory_region_set_log: Turn dirty logging on or off for a region.
*
* Turns dirty logging on or off for a specified client (display, migration).
@@ -686,8 +689,9 @@ void memory_region_transaction_commit(void);
* space
*
* @listener: an object containing the callbacks to be called
+ * @filter: if non-%NULL, only regions in this address space will be observed
*/
-void memory_listener_register(MemoryListener *listener);
+void memory_listener_register(MemoryListener *listener, MemoryRegion *filter);
/**
* memory_listener_unregister: undo the effect of memory_listener_register()
diff --git a/xen-all.c b/xen-all.c
index b0ed1ed193..6cef5062bc 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -392,6 +392,14 @@ static void xen_set_memory(struct MemoryListener *listener,
}
}
+static void xen_begin(MemoryListener *listener)
+{
+}
+
+static void xen_commit(MemoryListener *listener)
+{
+}
+
static void xen_region_add(MemoryListener *listener,
MemoryRegionSection *section)
{
@@ -404,6 +412,11 @@ static void xen_region_del(MemoryListener *listener,
xen_set_memory(listener, section, false);
}
+static void xen_region_nop(MemoryListener *listener,
+ MemoryRegionSection *section)
+{
+}
+
static void xen_sync_dirty_bitmap(XenIOState *state,
target_phys_addr_t start_addr,
ram_addr_t size)
@@ -485,14 +498,32 @@ static void xen_log_global_stop(MemoryListener *listener)
{
}
+static void xen_eventfd_add(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+}
+
+static void xen_eventfd_del(MemoryListener *listener,
+ MemoryRegionSection *section,
+ bool match_data, uint64_t data, int fd)
+{
+}
+
static MemoryListener xen_memory_listener = {
+ .begin = xen_begin,
+ .commit = xen_commit,
.region_add = xen_region_add,
.region_del = xen_region_del,
+ .region_nop = xen_region_nop,
.log_start = xen_log_start,
.log_stop = xen_log_stop,
.log_sync = xen_log_sync,
.log_global_start = xen_log_global_start,
.log_global_stop = xen_log_global_stop,
+ .eventfd_add = xen_eventfd_add,
+ .eventfd_del = xen_eventfd_del,
+ .priority = 10,
};
/* VCPU Operations, MMIO, IO ring ... */
@@ -975,7 +1006,7 @@ int xen_hvm_init(void)
state->memory_listener = xen_memory_listener;
QLIST_INIT(&state->physmap);
- memory_listener_register(&state->memory_listener);
+ memory_listener_register(&state->memory_listener, get_system_memory());
state->log_for_dirtybit = NULL;
/* Initialize backend core & drivers */
generated by cgit v1.2.3 (git 2.26.2) at 2024-12-03 03:03:18 +0000