diff options
Diffstat (limited to 'tcg')
| -rw-r--r-- | tcg/aarch64/tcg-target.inc.c | 13 | ||||
| -rw-r--r-- | tcg/arm/tcg-target.inc.c | 19 | ||||
| -rw-r--r-- | tcg/i386/tcg-target.inc.c | 19 | ||||
| -rw-r--r-- | tcg/ia64/tcg-target.inc.c | 22 | ||||
| -rw-r--r-- | tcg/mips/tcg-target.inc.c | 11 | ||||
| -rw-r--r-- | tcg/ppc/tcg-target.inc.c | 58 | ||||
| -rw-r--r-- | tcg/s390/tcg-target.inc.c | 15 | ||||
| -rw-r--r-- | tcg/sparc/tcg-target.inc.c | 16 | ||||
| -rw-r--r-- | tcg/tcg.h | 51 |
9 files changed, 120 insertions, 104 deletions
diff --git a/tcg/aarch64/tcg-target.inc.c b/tcg/aarch64/tcg-target.inc.c index 08b2d031aa..c8374b864f 100644 --- a/tcg/aarch64/tcg-target.inc.c +++ b/tcg/aarch64/tcg-target.inc.c @@ -1081,23 +1081,24 @@ static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg, TCGMemOp opc, int tlb_offset = is_read ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read) : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write); - int a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + unsigned a_mask = (1u << a_bits) - 1; + unsigned s_mask = (1u << s_bits) - 1; TCGReg base = TCG_AREG0, x3; uint64_t tlb_mask; /* For aligned accesses, we check the first byte and include the alignment bits within the address. For unaligned access, we check that we don't cross pages using the address of the last byte of the access. */ - if (a_bits >= 0) { - /* A byte access or an alignment check required */ - tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1); + if (a_bits >= s_bits) { x3 = addr_reg; } else { tcg_out_insn(s, 3401, ADDI, TARGET_LONG_BITS == 64, - TCG_REG_X3, addr_reg, (1 << (opc & MO_SIZE)) - 1); - tlb_mask = TARGET_PAGE_MASK; + TCG_REG_X3, addr_reg, s_mask - a_mask); x3 = TCG_REG_X3; } + tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask; /* Extract the TLB index from the address into X0. X0<CPU_TLB_BITS:0> = diff --git a/tcg/arm/tcg-target.inc.c b/tcg/arm/tcg-target.inc.c index 172febafdd..094f3f804d 100644 --- a/tcg/arm/tcg-target.inc.c +++ b/tcg/arm/tcg-target.inc.c @@ -1168,7 +1168,7 @@ QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1]) containing the addend of the tlb entry. Clobbers R0, R1, R2, TMP. */ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, - TCGMemOp s_bits, int mem_index, bool is_load) + TCGMemOp opc, int mem_index, bool is_load) { TCGReg base = TCG_AREG0; int cmp_off = @@ -1176,6 +1176,8 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, ? offsetof(CPUArchState, tlb_table[mem_index][0].addr_read) : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write)); int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend); + unsigned s_bits = opc & MO_SIZE; + unsigned a_bits = get_alignment_bits(opc); /* Should generate something like the following: * shr tmp, addrlo, #TARGET_PAGE_BITS (1) @@ -1216,10 +1218,13 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGReg addrlo, TCGReg addrhi, } } - /* Check alignment. */ - if (s_bits) { - tcg_out_dat_imm(s, COND_AL, ARITH_TST, - 0, addrlo, (1 << s_bits) - 1); + /* Check alignment. We don't support inline unaligned acceses, + but we can easily support overalignment checks. */ + if (a_bits < s_bits) { + a_bits = s_bits; + } + if (a_bits) { + tcg_out_dat_imm(s, COND_AL, ARITH_TST, 0, addrlo, (1 << a_bits) - 1); } /* Load the tlb addend. */ @@ -1499,7 +1504,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, bool is64) #ifdef CONFIG_SOFTMMU mem_index = get_mmuidx(oi); - addend = tcg_out_tlb_read(s, addrlo, addrhi, opc & MO_SIZE, mem_index, 1); + addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, mem_index, 1); /* This a conditional BL only to load a pointer within this opcode into LR for the slow path. We will not be using the value for a tail call. */ @@ -1630,7 +1635,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, bool is64) #ifdef CONFIG_SOFTMMU mem_index = get_mmuidx(oi); - addend = tcg_out_tlb_read(s, addrlo, addrhi, opc & MO_SIZE, mem_index, 0); + addend = tcg_out_tlb_read(s, addrlo, addrhi, opc, mem_index, 0); tcg_out_qemu_st_index(s, COND_EQ, opc, datalo, datahi, addrlo, addend); diff --git a/tcg/i386/tcg-target.inc.c b/tcg/i386/tcg-target.inc.c index 6f8cdca756..e30a122f19 100644 --- a/tcg/i386/tcg-target.inc.c +++ b/tcg/i386/tcg-target.inc.c @@ -1202,7 +1202,10 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, TCGType ttype = TCG_TYPE_I32; TCGType tlbtype = TCG_TYPE_I32; int trexw = 0, hrexw = 0, tlbrexw = 0; - int a_bits = get_alignment_bits(opc); + unsigned a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + unsigned a_mask = (1 << a_bits) - 1; + unsigned s_mask = (1 << s_bits) - 1; target_ulong tlb_mask; if (TCG_TARGET_REG_BITS == 64) { @@ -1220,17 +1223,15 @@ static inline void tcg_out_tlb_load(TCGContext *s, TCGReg addrlo, TCGReg addrhi, } tcg_out_mov(s, tlbtype, r0, addrlo); - if (a_bits >= 0) { - /* A byte access or an alignment check required */ + /* If the required alignment is at least as large as the access, simply + copy the address and mask. For lesser alignments, check that we don't + cross pages for the complete access. */ + if (a_bits >= s_bits) { tcg_out_mov(s, ttype, r1, addrlo); - tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1); } else { - /* For unaligned access check that we don't cross pages using - the page address of the last byte. */ - tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, - (1 << (opc & MO_SIZE)) - 1); - tlb_mask = TARGET_PAGE_MASK; + tcg_out_modrm_offset(s, OPC_LEA + trexw, r1, addrlo, s_mask - a_mask); } + tlb_mask = TARGET_PAGE_MASK | a_mask; tcg_out_shifti(s, SHIFT_SHR + tlbrexw, r0, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS); diff --git a/tcg/ia64/tcg-target.inc.c b/tcg/ia64/tcg-target.inc.c index c91f39281b..7642390b4b 100644 --- a/tcg/ia64/tcg-target.inc.c +++ b/tcg/ia64/tcg-target.inc.c @@ -1496,10 +1496,18 @@ QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1]) R1, R3 are clobbered, leaving R56 free for... BSWAP_1, BSWAP_2 and I-slot insns for swapping data for store. */ static inline void tcg_out_qemu_tlb(TCGContext *s, TCGReg addr_reg, - TCGMemOp s_bits, int off_rw, int off_add, + TCGMemOp opc, int off_rw, int off_add, uint64_t bswap1, uint64_t bswap2) { - /* + unsigned s_bits = opc & MO_SIZE; + unsigned a_bits = get_alignment_bits(opc); + + /* We don't support unaligned accesses, but overalignment is easy. */ + if (a_bits < s_bits) { + a_bits = s_bits; + } + + /* .mii mov r2 = off_rw extr.u r3 = addr_reg, ... # extract tlb page @@ -1521,7 +1529,7 @@ static inline void tcg_out_qemu_tlb(TCGContext *s, TCGReg addr_reg, cmp.eq p6, p7 = r3, r58 nop ;; - */ + */ tcg_out_bundle(s, miI, tcg_opc_movi_a(TCG_REG_P0, TCG_REG_R2, off_rw), tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, TCG_REG_R3, @@ -1536,8 +1544,8 @@ static inline void tcg_out_qemu_tlb(TCGContext *s, TCGReg addr_reg, TCG_REG_R3, 63 - CPU_TLB_ENTRY_BITS, 63 - CPU_TLB_ENTRY_BITS), tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, TCG_REG_R1, 0, - TCG_REG_R57, 63 - s_bits, - TARGET_PAGE_BITS - s_bits - 1)); + TCG_REG_R57, 63 - a_bits, + TARGET_PAGE_BITS - a_bits - 1)); tcg_out_bundle(s, MmI, tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2, TCG_REG_R2, TCG_REG_R3), @@ -1661,7 +1669,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args) s_bits = opc & MO_SIZE; /* Read the TLB entry */ - tcg_out_qemu_tlb(s, addr_reg, s_bits, + tcg_out_qemu_tlb(s, addr_reg, opc, offsetof(CPUArchState, tlb_table[mem_index][0].addr_read), offsetof(CPUArchState, tlb_table[mem_index][0].addend), INSN_NOP_I, INSN_NOP_I); @@ -1739,7 +1747,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args) pre1 = tcg_opc_ext_i(TCG_REG_P0, opc, TCG_REG_R58, data_reg); } - tcg_out_qemu_tlb(s, addr_reg, s_bits, + tcg_out_qemu_tlb(s, addr_reg, opc, offsetof(CPUArchState, tlb_table[mem_index][0].addr_write), offsetof(CPUArchState, tlb_table[mem_index][0].addend), pre1, pre2); diff --git a/tcg/mips/tcg-target.inc.c b/tcg/mips/tcg-target.inc.c index 2f9be48139..acb6ff06c6 100644 --- a/tcg/mips/tcg-target.inc.c +++ b/tcg/mips/tcg-target.inc.c @@ -1040,7 +1040,9 @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, TCGReg addrh, TCGMemOpIdx oi, tcg_insn_unit *label_ptr[2], bool is_load) { - TCGMemOp s_bits = get_memop(oi) & MO_SIZE; + TCGMemOp opc = get_memop(oi); + unsigned s_bits = opc & MO_SIZE; + unsigned a_bits = get_alignment_bits(opc); int mem_index = get_mmuidx(oi); int cmp_off = (is_load @@ -1071,10 +1073,15 @@ static void tcg_out_tlb_load(TCGContext *s, TCGReg base, TCGReg addrl, tcg_out_opc_imm(s, OPC_LW, TCG_TMP0, TCG_REG_A0, cmp_off + (TARGET_LONG_BITS == 64 ? LO_OFF : 0)); + /* We don't currently support unaligned accesses. + We could do so with mips32r6. */ + if (a_bits < s_bits) { + a_bits = s_bits; + } /* Mask the page bits, keeping the alignment bits to compare against. In between on 32-bit targets, load the tlb addend for the fast path. */ tcg_out_movi(s, TCG_TYPE_I32, TCG_TMP1, - TARGET_PAGE_MASK | ((1 << s_bits) - 1)); + TARGET_PAGE_MASK | ((1 << a_bits) - 1)); if (TARGET_LONG_BITS == 32) { tcg_out_opc_imm(s, OPC_LW, TCG_REG_A0, TCG_REG_A0, add_off); } diff --git a/tcg/ppc/tcg-target.inc.c b/tcg/ppc/tcg-target.inc.c index eaf1bd9bfd..d79969096f 100644 --- a/tcg/ppc/tcg-target.inc.c +++ b/tcg/ppc/tcg-target.inc.c @@ -1404,8 +1404,8 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc, : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write)); int add_off = offsetof(CPUArchState, tlb_table[mem_index][0].addend); TCGReg base = TCG_AREG0; - TCGMemOp s_bits = opc & MO_SIZE; - int a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + unsigned a_bits = get_alignment_bits(opc); /* Extract the page index, shifted into place for tlb index. */ if (TCG_TARGET_REG_BITS == 64) { @@ -1458,39 +1458,43 @@ static TCGReg tcg_out_tlb_read(TCGContext *s, TCGMemOp opc, tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R3, TCG_REG_R3, add_off); /* Clear the non-page, non-alignment bits from the address */ - if (TCG_TARGET_REG_BITS == 32 || TARGET_LONG_BITS == 32) { - /* We don't support unaligned accesses on 32-bits, preserve - * the bottom bits and thus trigger a comparison failure on - * unaligned accesses + if (TCG_TARGET_REG_BITS == 32) { + /* We don't support unaligned accesses on 32-bits. + * Preserve the bottom bits and thus trigger a comparison + * failure on unaligned accesses. */ - if (a_bits < 0) { + if (a_bits < s_bits) { a_bits = s_bits; } tcg_out_rlw(s, RLWINM, TCG_REG_R0, addrlo, 0, (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS); - } else if (a_bits) { - /* More than byte access, we need to handle alignment */ - if (a_bits > 0) { - /* Alignment required by the front-end, same as 32-bits */ - tcg_out_rld(s, RLDICL, TCG_REG_R0, addrlo, + } else { + TCGReg t = addrlo; + + /* If the access is unaligned, we need to make sure we fail if we + * cross a page boundary. The trick is to add the access size-1 + * to the address before masking the low bits. That will make the + * address overflow to the next page if we cross a page boundary, + * which will then force a mismatch of the TLB compare. + */ + if (a_bits < s_bits) { + unsigned a_mask = (1 << a_bits) - 1; + unsigned s_mask = (1 << s_bits) - 1; + tcg_out32(s, ADDI | TAI(TCG_REG_R0, t, s_mask - a_mask)); + t = TCG_REG_R0; + } + + /* Mask the address for the requested alignment. */ + if (TARGET_LONG_BITS == 32) { + tcg_out_rlw(s, RLWINM, TCG_REG_R0, t, 0, + (32 - a_bits) & 31, 31 - TARGET_PAGE_BITS); + } else if (a_bits == 0) { + tcg_out_rld(s, RLDICR, TCG_REG_R0, t, 0, 63 - TARGET_PAGE_BITS); + } else { + tcg_out_rld(s, RLDICL, TCG_REG_R0, t, 64 - TARGET_PAGE_BITS, TARGET_PAGE_BITS - a_bits); tcg_out_rld(s, RLDICL, TCG_REG_R0, TCG_REG_R0, TARGET_PAGE_BITS, 0); - } else { - /* We support unaligned accesses, we need to make sure we fail - * if we cross a page boundary. The trick is to add the - * access_size-1 to the address before masking the low bits. - * That will make the address overflow to the next page if we - * cross a page boundary which will then force a mismatch of - * the TLB compare since the next page cannot possibly be in - * the same TLB index. - */ - tcg_out32(s, ADDI | TAI(TCG_REG_R0, addrlo, (1 << s_bits) - 1)); - tcg_out_rld(s, RLDICR, TCG_REG_R0, TCG_REG_R0, - 0, 63 - TARGET_PAGE_BITS); } - } else { - /* Byte access, just chop off the bits below the page index */ - tcg_out_rld(s, RLDICR, TCG_REG_R0, addrlo, 0, 63 - TARGET_PAGE_BITS); } if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { diff --git a/tcg/s390/tcg-target.inc.c b/tcg/s390/tcg-target.inc.c index 5a7495b063..f0c88de6d3 100644 --- a/tcg/s390/tcg-target.inc.c +++ b/tcg/s390/tcg-target.inc.c @@ -1505,21 +1505,18 @@ QEMU_BUILD_BUG_ON(offsetof(CPUArchState, tlb_table[NB_MMU_MODES - 1][1]) static TCGReg tcg_out_tlb_read(TCGContext* s, TCGReg addr_reg, TCGMemOp opc, int mem_index, bool is_ld) { - int a_bits = get_alignment_bits(opc); + unsigned s_bits = opc & MO_SIZE; + unsigned a_bits = get_alignment_bits(opc); + unsigned s_mask = (1 << s_bits) - 1; + unsigned a_mask = (1 << a_bits) - 1; int ofs, a_off; uint64_t tlb_mask; /* For aligned accesses, we check the first byte and include the alignment bits within the address. For unaligned access, we check that we don't cross pages using the address of the last byte of the access. */ - if (a_bits >= 0) { - /* A byte access or an alignment check required */ - a_off = 0; - tlb_mask = TARGET_PAGE_MASK | ((1 << a_bits) - 1); - } else { - a_off = (1 << (opc & MO_SIZE)) - 1; - tlb_mask = TARGET_PAGE_MASK; - } + a_off = (a_bits >= s_bits ? 0 : s_mask - a_mask); + tlb_mask = (uint64_t)TARGET_PAGE_MASK | a_mask; if (facilities & FACILITY_GEN_INST_EXT) { tcg_out_risbg(s, TCG_REG_R2, addr_reg, diff --git a/tcg/sparc/tcg-target.inc.c b/tcg/sparc/tcg-target.inc.c index 8e98172ca0..92f8818a9e 100644 --- a/tcg/sparc/tcg-target.inc.c +++ b/tcg/sparc/tcg-target.inc.c @@ -996,19 +996,25 @@ static void tcg_target_qemu_prologue(TCGContext *s) is in the returned register, maybe %o0. The TLB addend is in %o1. */ static TCGReg tcg_out_tlb_load(TCGContext *s, TCGReg addr, int mem_index, - TCGMemOp s_bits, int which) + TCGMemOp opc, int which) { const TCGReg r0 = TCG_REG_O0; const TCGReg r1 = TCG_REG_O1; const TCGReg r2 = TCG_REG_O2; + unsigned s_bits = opc & MO_SIZE; + unsigned a_bits = get_alignment_bits(opc); int tlb_ofs; /* Shift the page number down. */ tcg_out_arithi(s, r1, addr, TARGET_PAGE_BITS, SHIFT_SRL); - /* Mask out the page offset, except for the required alignment. */ + /* Mask out the page offset, except for the required alignment. + We don't support unaligned accesses. */ + if (a_bits < s_bits) { + a_bits = s_bits; + } tcg_out_movi(s, TCG_TYPE_TL, TCG_REG_T1, - TARGET_PAGE_MASK | ((1 << s_bits) - 1)); + TARGET_PAGE_MASK | ((1 << a_bits) - 1)); /* Mask the tlb index. */ tcg_out_arithi(s, r1, r1, CPU_TLB_SIZE - 1, ARITH_AND); @@ -1087,7 +1093,7 @@ static void tcg_out_qemu_ld(TCGContext *s, TCGReg data, TCGReg addr, tcg_insn_unit *func; tcg_insn_unit *label_ptr; - addrz = tcg_out_tlb_load(s, addr, memi, memop & MO_SIZE, + addrz = tcg_out_tlb_load(s, addr, memi, memop, offsetof(CPUTLBEntry, addr_read)); /* The fast path is exactly one insn. Thus we can perform the @@ -1169,7 +1175,7 @@ static void tcg_out_qemu_st(TCGContext *s, TCGReg data, TCGReg addr, tcg_insn_unit *func; tcg_insn_unit *label_ptr; - addrz = tcg_out_tlb_load(s, addr, memi, memop & MO_SIZE, + addrz = tcg_out_tlb_load(s, addr, memi, memop, offsetof(CPUTLBEntry, addr_write)); /* The fast path is exactly one insn. Thus we can perform the entire @@ -287,20 +287,19 @@ typedef enum TCGMemOp { * MO_ALIGN accesses will result in a call to the CPU's * do_unaligned_access hook if the guest address is not aligned. * The default depends on whether the target CPU defines ALIGNED_ONLY. + * * Some architectures (e.g. ARMv8) need the address which is aligned * to a size more than the size of the memory access. - * To support such check it's enough the current costless alignment - * check implementation in QEMU, but we need to support - * an alignment size specifying. - * MO_ALIGN supposes a natural alignment - * (i.e. the alignment size is the size of a memory access). - * Note that an alignment size must be equal or greater - * than an access size. + * Some architectures (e.g. SPARCv9) need an address which is aligned, + * but less strictly than the natural alignment. + * + * MO_ALIGN supposes the alignment size is the size of a memory access. + * * There are three options: - * - an alignment to the size of an access (MO_ALIGN); - * - an alignment to the specified size that is equal or greater than - * an access size (MO_ALIGN_x where 'x' is a size in bytes); * - unaligned access permitted (MO_UNALN). + * - an alignment to the size of an access (MO_ALIGN); + * - an alignment to a specified size, which may be more or less than + * the access size (MO_ALIGN_x where 'x' is a size in bytes); */ MO_ASHIFT = 4, MO_AMASK = 7 << MO_ASHIFT, @@ -353,38 +352,26 @@ typedef enum TCGMemOp { * @memop: TCGMemOp value * * Extract the alignment size from the memop. - * - * Returns: 0 in case of byte access (which is always aligned); - * positive value - number of alignment bits; - * negative value if unaligned access enabled - * and this is not a byte access. */ -static inline int get_alignment_bits(TCGMemOp memop) +static inline unsigned get_alignment_bits(TCGMemOp memop) { - int a = memop & MO_AMASK; - int s = memop & MO_SIZE; - int r; + unsigned a = memop & MO_AMASK; if (a == MO_UNALN) { - /* Negative value if unaligned access enabled, - * or zero value in case of byte access. - */ - return -s; + /* No alignment required. */ + a = 0; } else if (a == MO_ALIGN) { - /* A natural alignment: return a number of access size bits */ - r = s; + /* A natural alignment requirement. */ + a = memop & MO_SIZE; } else { - /* Specific alignment size. It must be equal or greater - * than the access size. - */ - r = a >> MO_ASHIFT; - tcg_debug_assert(r >= s); + /* A specific alignment requirement. */ + a = a >> MO_ASHIFT; } #if defined(CONFIG_SOFTMMU) /* The requested alignment cannot overlap the TLB flags. */ - tcg_debug_assert((TLB_FLAGS_MASK & ((1 << r) - 1)) == 0); + tcg_debug_assert((TLB_FLAGS_MASK & ((1 << a) - 1)) == 0); #endif - return r; + return a; } typedef tcg_target_ulong TCGArg; |