1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
/*
* SPDX-License-Identifier: GPL-2.0-or-later
* Load/store for 128-bit atomic operations, x86_64 version.
*
* Copyright (C) 2023 Linaro, Ltd.
*
* See docs/devel/atomics.rst for discussion about the guarantees each
* atomic primitive is meant to provide.
*/
#ifndef AARCH64_ATOMIC128_LDST_H
#define AARCH64_ATOMIC128_LDST_H
#ifdef CONFIG_INT128_TYPE
#include "host/cpuinfo.h"
#include "tcg/debug-assert.h"
/*
* Through clang 16, with -mcx16, __atomic_load_n is incorrectly
* expanded to a read-write operation: lock cmpxchg16b.
*/
#define HAVE_ATOMIC128_RO likely(cpuinfo & CPUINFO_ATOMIC_VMOVDQA)
#define HAVE_ATOMIC128_RW 1
static inline Int128 atomic16_read_ro(const Int128 *ptr)
{
Int128Alias r;
tcg_debug_assert(HAVE_ATOMIC128_RO);
asm("vmovdqa %1, %0" : "=x" (r.i) : "m" (*ptr));
return r.s;
}
static inline Int128 atomic16_read_rw(Int128 *ptr)
{
__int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
Int128Alias r;
if (HAVE_ATOMIC128_RO) {
asm("vmovdqa %1, %0" : "=x" (r.i) : "m" (*ptr_align));
} else {
r.i = __sync_val_compare_and_swap_16(ptr_align, 0, 0);
}
return r.s;
}
static inline void atomic16_set(Int128 *ptr, Int128 val)
{
__int128_t *ptr_align = __builtin_assume_aligned(ptr, 16);
Int128Alias new = { .s = val };
if (HAVE_ATOMIC128_RO) {
asm("vmovdqa %1, %0" : "=m"(*ptr_align) : "x" (new.i));
} else {
__int128_t old;
do {
old = *ptr_align;
} while (!__sync_bool_compare_and_swap_16(ptr_align, old, new.i));
}
}
#else
/* Provide QEMU_ERROR stubs. */
#include "host/include/generic/host/atomic128-ldst.h"
#endif
#endif /* AARCH64_ATOMIC128_LDST_H */
|
