aboutsummaryrefslogtreecommitdiff
path: root/accel/tcg/ldst_atomicity.c.inc
diff options
context:
space:
mode:
Diffstat (limited to 'accel/tcg/ldst_atomicity.c.inc')
-rw-r--r--accel/tcg/ldst_atomicity.c.inc491
1 files changed, 491 insertions, 0 deletions
diff --git a/accel/tcg/ldst_atomicity.c.inc b/accel/tcg/ldst_atomicity.c.inc
index a5bce641f8..1f39e43896 100644
--- a/accel/tcg/ldst_atomicity.c.inc
+++ b/accel/tcg/ldst_atomicity.c.inc
@@ -21,6 +21,12 @@
#else
# define HAVE_al16_fast false
#endif
+#if defined(CONFIG_ATOMIC128) || defined(CONFIG_CMPXCHG128)
+# define HAVE_al16 true
+#else
+# define HAVE_al16 false
+#endif
+
/**
* required_atomicity:
@@ -564,3 +570,488 @@ static uint64_t load_atom_8(CPUArchState *env, uintptr_t ra,
g_assert_not_reached();
}
}
+
+/**
+ * store_atomic2:
+ * @pv: host address
+ * @val: value to store
+ *
+ * Atomically store 2 aligned bytes to @pv.
+ */
+static inline void store_atomic2(void *pv, uint16_t val)
+{
+ uint16_t *p = __builtin_assume_aligned(pv, 2);
+ qatomic_set(p, val);
+}
+
+/**
+ * store_atomic4:
+ * @pv: host address
+ * @val: value to store
+ *
+ * Atomically store 4 aligned bytes to @pv.
+ */
+static inline void store_atomic4(void *pv, uint32_t val)
+{
+ uint32_t *p = __builtin_assume_aligned(pv, 4);
+ qatomic_set(p, val);
+}
+
+/**
+ * store_atomic8:
+ * @pv: host address
+ * @val: value to store
+ *
+ * Atomically store 8 aligned bytes to @pv.
+ */
+static inline void store_atomic8(void *pv, uint64_t val)
+{
+ uint64_t *p = __builtin_assume_aligned(pv, 8);
+
+ qemu_build_assert(HAVE_al8);
+ qatomic_set__nocheck(p, val);
+}
+
+/**
+ * store_atom_4x2
+ */
+static inline void store_atom_4_by_2(void *pv, uint32_t val)
+{
+ store_atomic2(pv, val >> (HOST_BIG_ENDIAN ? 16 : 0));
+ store_atomic2(pv + 2, val >> (HOST_BIG_ENDIAN ? 0 : 16));
+}
+
+/**
+ * store_atom_8_by_2
+ */
+static inline void store_atom_8_by_2(void *pv, uint64_t val)
+{
+ store_atom_4_by_2(pv, val >> (HOST_BIG_ENDIAN ? 32 : 0));
+ store_atom_4_by_2(pv + 4, val >> (HOST_BIG_ENDIAN ? 0 : 32));
+}
+
+/**
+ * store_atom_8_by_4
+ */
+static inline void store_atom_8_by_4(void *pv, uint64_t val)
+{
+ store_atomic4(pv, val >> (HOST_BIG_ENDIAN ? 32 : 0));
+ store_atomic4(pv + 4, val >> (HOST_BIG_ENDIAN ? 0 : 32));
+}
+
+/**
+ * store_atom_insert_al4:
+ * @p: host address
+ * @val: shifted value to store
+ * @msk: mask for value to store
+ *
+ * Atomically store @val to @p, masked by @msk.
+ */
+static void store_atom_insert_al4(uint32_t *p, uint32_t val, uint32_t msk)
+{
+ uint32_t old, new;
+
+ p = __builtin_assume_aligned(p, 4);
+ old = qatomic_read(p);
+ do {
+ new = (old & ~msk) | val;
+ } while (!__atomic_compare_exchange_n(p, &old, new, true,
+ __ATOMIC_RELAXED, __ATOMIC_RELAXED));
+}
+
+/**
+ * store_atom_insert_al8:
+ * @p: host address
+ * @val: shifted value to store
+ * @msk: mask for value to store
+ *
+ * Atomically store @val to @p masked by @msk.
+ */
+static void store_atom_insert_al8(uint64_t *p, uint64_t val, uint64_t msk)
+{
+ uint64_t old, new;
+
+ qemu_build_assert(HAVE_al8);
+ p = __builtin_assume_aligned(p, 8);
+ old = qatomic_read__nocheck(p);
+ do {
+ new = (old & ~msk) | val;
+ } while (!__atomic_compare_exchange_n(p, &old, new, true,
+ __ATOMIC_RELAXED, __ATOMIC_RELAXED));
+}
+
+/**
+ * store_atom_insert_al16:
+ * @p: host address
+ * @val: shifted value to store
+ * @msk: mask for value to store
+ *
+ * Atomically store @val to @p masked by @msk.
+ */
+static void store_atom_insert_al16(Int128 *ps, Int128Alias val, Int128Alias msk)
+{
+#if defined(CONFIG_ATOMIC128)
+ __uint128_t *pu, old, new;
+
+ /* With CONFIG_ATOMIC128, we can avoid the memory barriers. */
+ pu = __builtin_assume_aligned(ps, 16);
+ old = *pu;
+ do {
+ new = (old & ~msk.u) | val.u;
+ } while (!__atomic_compare_exchange_n(pu, &old, new, true,
+ __ATOMIC_RELAXED, __ATOMIC_RELAXED));
+#elif defined(CONFIG_CMPXCHG128)
+ __uint128_t *pu, old, new;
+
+ /*
+ * Without CONFIG_ATOMIC128, __atomic_compare_exchange_n will always
+ * defer to libatomic, so we must use __sync_*_compare_and_swap_16
+ * and accept the sequential consistency that comes with it.
+ */
+ pu = __builtin_assume_aligned(ps, 16);
+ do {
+ old = *pu;
+ new = (old & ~msk.u) | val.u;
+ } while (!__sync_bool_compare_and_swap_16(pu, old, new));
+#else
+ qemu_build_not_reached();
+#endif
+}
+
+/**
+ * store_bytes_leN:
+ * @pv: host address
+ * @size: number of bytes to store
+ * @val_le: data to store
+ *
+ * Store @size bytes at @p. The bytes to store are extracted in little-endian order
+ * from @val_le; return the bytes of @val_le beyond @size that have not been stored.
+ */
+static uint64_t store_bytes_leN(void *pv, int size, uint64_t val_le)
+{
+ uint8_t *p = pv;
+ for (int i = 0; i < size; i++, val_le >>= 8) {
+ p[i] = val_le;
+ }
+ return val_le;
+}
+
+/**
+ * store_parts_leN
+ * @pv: host address
+ * @size: number of bytes to store
+ * @val_le: data to store
+ *
+ * As store_bytes_leN, but atomically on each aligned part.
+ */
+G_GNUC_UNUSED
+static uint64_t store_parts_leN(void *pv, int size, uint64_t val_le)
+{
+ do {
+ int n;
+
+ /* Find minimum of alignment and size */
+ switch (((uintptr_t)pv | size) & 7) {
+ case 4:
+ store_atomic4(pv, le32_to_cpu(val_le));
+ val_le >>= 32;
+ n = 4;
+ break;
+ case 2:
+ case 6:
+ store_atomic2(pv, le16_to_cpu(val_le));
+ val_le >>= 16;
+ n = 2;
+ break;
+ default:
+ *(uint8_t *)pv = val_le;
+ val_le >>= 8;
+ n = 1;
+ break;
+ case 0:
+ g_assert_not_reached();
+ }
+ pv += n;
+ size -= n;
+ } while (size != 0);
+
+ return val_le;
+}
+
+/**
+ * store_whole_le4
+ * @pv: host address
+ * @size: number of bytes to store
+ * @val_le: data to store
+ *
+ * As store_bytes_leN, but atomically as a whole.
+ * Four aligned bytes are guaranteed to cover the store.
+ */
+static uint64_t store_whole_le4(void *pv, int size, uint64_t val_le)
+{
+ int sz = size * 8;
+ int o = (uintptr_t)pv & 3;
+ int sh = o * 8;
+ uint32_t m = MAKE_64BIT_MASK(0, sz);
+ uint32_t v;
+
+ if (HOST_BIG_ENDIAN) {
+ v = bswap32(val_le) >> sh;
+ m = bswap32(m) >> sh;
+ } else {
+ v = val_le << sh;
+ m <<= sh;
+ }
+ store_atom_insert_al4(pv - o, v, m);
+ return val_le >> sz;
+}
+
+/**
+ * store_whole_le8
+ * @pv: host address
+ * @size: number of bytes to store
+ * @val_le: data to store
+ *
+ * As store_bytes_leN, but atomically as a whole.
+ * Eight aligned bytes are guaranteed to cover the store.
+ */
+static uint64_t store_whole_le8(void *pv, int size, uint64_t val_le)
+{
+ int sz = size * 8;
+ int o = (uintptr_t)pv & 7;
+ int sh = o * 8;
+ uint64_t m = MAKE_64BIT_MASK(0, sz);
+ uint64_t v;
+
+ qemu_build_assert(HAVE_al8);
+ if (HOST_BIG_ENDIAN) {
+ v = bswap64(val_le) >> sh;
+ m = bswap64(m) >> sh;
+ } else {
+ v = val_le << sh;
+ m <<= sh;
+ }
+ store_atom_insert_al8(pv - o, v, m);
+ return val_le >> sz;
+}
+
+/**
+ * store_whole_le16
+ * @pv: host address
+ * @size: number of bytes to store
+ * @val_le: data to store
+ *
+ * As store_bytes_leN, but atomically as a whole.
+ * 16 aligned bytes are guaranteed to cover the store.
+ */
+static uint64_t store_whole_le16(void *pv, int size, Int128 val_le)
+{
+ int sz = size * 8;
+ int o = (uintptr_t)pv & 15;
+ int sh = o * 8;
+ Int128 m, v;
+
+ qemu_build_assert(HAVE_al16);
+
+ /* Like MAKE_64BIT_MASK(0, sz), but larger. */
+ if (sz <= 64) {
+ m = int128_make64(MAKE_64BIT_MASK(0, sz));
+ } else {
+ m = int128_make128(-1, MAKE_64BIT_MASK(0, sz - 64));
+ }
+
+ if (HOST_BIG_ENDIAN) {
+ v = int128_urshift(bswap128(val_le), sh);
+ m = int128_urshift(bswap128(m), sh);
+ } else {
+ v = int128_lshift(val_le, sh);
+ m = int128_lshift(m, sh);
+ }
+ store_atom_insert_al16(pv - o, v, m);
+
+ /* Unused if sz <= 64. */
+ return int128_gethi(val_le) >> (sz - 64);
+}
+
+/**
+ * store_atom_2:
+ * @p: host address
+ * @val: the value to store
+ * @memop: the full memory op
+ *
+ * Store 2 bytes to @p, honoring the atomicity of @memop.
+ */
+static void store_atom_2(CPUArchState *env, uintptr_t ra,
+ void *pv, MemOp memop, uint16_t val)
+{
+ uintptr_t pi = (uintptr_t)pv;
+ int atmax;
+
+ if (likely((pi & 1) == 0)) {
+ store_atomic2(pv, val);
+ return;
+ }
+
+ atmax = required_atomicity(env, pi, memop);
+ if (atmax == MO_8) {
+ stw_he_p(pv, val);
+ return;
+ }
+
+ /*
+ * The only case remaining is MO_ATOM_WITHIN16.
+ * Big or little endian, we want the middle two bytes in each test.
+ */
+ if ((pi & 3) == 1) {
+ store_atom_insert_al4(pv - 1, (uint32_t)val << 8, MAKE_64BIT_MASK(8, 16));
+ return;
+ } else if ((pi & 7) == 3) {
+ if (HAVE_al8) {
+ store_atom_insert_al8(pv - 3, (uint64_t)val << 24, MAKE_64BIT_MASK(24, 16));
+ return;
+ }
+ } else if ((pi & 15) == 7) {
+ if (HAVE_al16) {
+ Int128 v = int128_lshift(int128_make64(val), 56);
+ Int128 m = int128_lshift(int128_make64(0xffff), 56);
+ store_atom_insert_al16(pv - 7, v, m);
+ return;
+ }
+ } else {
+ g_assert_not_reached();
+ }
+
+ cpu_loop_exit_atomic(env_cpu(env), ra);
+}
+
+/**
+ * store_atom_4:
+ * @p: host address
+ * @val: the value to store
+ * @memop: the full memory op
+ *
+ * Store 4 bytes to @p, honoring the atomicity of @memop.
+ */
+static void store_atom_4(CPUArchState *env, uintptr_t ra,
+ void *pv, MemOp memop, uint32_t val)
+{
+ uintptr_t pi = (uintptr_t)pv;
+ int atmax;
+
+ if (likely((pi & 3) == 0)) {
+ store_atomic4(pv, val);
+ return;
+ }
+
+ atmax = required_atomicity(env, pi, memop);
+ switch (atmax) {
+ case MO_8:
+ stl_he_p(pv, val);
+ return;
+ case MO_16:
+ store_atom_4_by_2(pv, val);
+ return;
+ case -MO_16:
+ {
+ uint32_t val_le = cpu_to_le32(val);
+ int s2 = pi & 3;
+ int s1 = 4 - s2;
+
+ switch (s2) {
+ case 1:
+ val_le = store_whole_le4(pv, s1, val_le);
+ *(uint8_t *)(pv + 3) = val_le;
+ break;
+ case 3:
+ *(uint8_t *)pv = val_le;
+ store_whole_le4(pv + 1, s2, val_le >> 8);
+ break;
+ case 0: /* aligned */
+ case 2: /* atmax MO_16 */
+ default:
+ g_assert_not_reached();
+ }
+ }
+ return;
+ case MO_32:
+ if ((pi & 7) < 4) {
+ if (HAVE_al8) {
+ store_whole_le8(pv, 4, cpu_to_le32(val));
+ return;
+ }
+ } else {
+ if (HAVE_al16) {
+ store_whole_le16(pv, 4, int128_make64(cpu_to_le32(val)));
+ return;
+ }
+ }
+ cpu_loop_exit_atomic(env_cpu(env), ra);
+ default:
+ g_assert_not_reached();
+ }
+}
+
+/**
+ * store_atom_8:
+ * @p: host address
+ * @val: the value to store
+ * @memop: the full memory op
+ *
+ * Store 8 bytes to @p, honoring the atomicity of @memop.
+ */
+static void store_atom_8(CPUArchState *env, uintptr_t ra,
+ void *pv, MemOp memop, uint64_t val)
+{
+ uintptr_t pi = (uintptr_t)pv;
+ int atmax;
+
+ if (HAVE_al8 && likely((pi & 7) == 0)) {
+ store_atomic8(pv, val);
+ return;
+ }
+
+ atmax = required_atomicity(env, pi, memop);
+ switch (atmax) {
+ case MO_8:
+ stq_he_p(pv, val);
+ return;
+ case MO_16:
+ store_atom_8_by_2(pv, val);
+ return;
+ case MO_32:
+ store_atom_8_by_4(pv, val);
+ return;
+ case -MO_32:
+ if (HAVE_al8) {
+ uint64_t val_le = cpu_to_le64(val);
+ int s2 = pi & 7;
+ int s1 = 8 - s2;
+
+ switch (s2) {
+ case 1 ... 3:
+ val_le = store_whole_le8(pv, s1, val_le);
+ store_bytes_leN(pv + s1, s2, val_le);
+ break;
+ case 5 ... 7:
+ val_le = store_bytes_leN(pv, s1, val_le);
+ store_whole_le8(pv + s1, s2, val_le);
+ break;
+ case 0: /* aligned */
+ case 4: /* atmax MO_32 */
+ default:
+ g_assert_not_reached();
+ }
+ return;
+ }
+ break;
+ case MO_64:
+ if (HAVE_al16) {
+ store_whole_le16(pv, 8, int128_make64(cpu_to_le64(val)));
+ return;
+ }
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ cpu_loop_exit_atomic(env_cpu(env), ra);
+}
generated by cgit v1.2.3 (git 2.26.2) at 2024-12-22 16:53:21 +0000