80d4f39ed0244c43e9e9ecb82de233d38f61f9cb
[ghc.git] / libraries / ghc-prim / cbits / atomic.c
1 #include "Rts.h"
2
3 // Fallbacks for atomic primops on byte arrays. The builtins used
4 // below are supported on both GCC and LLVM.
5 //
6 // Ideally these function would take StgWord8, StgWord16, etc but
7 // older GCC versions incorrectly assume that the register that the
8 // argument is passed in has been zero extended, which is incorrect
9 // according to the ABI and is not what GHC does when it generates
10 // calls to these functions.
11
12 // FetchAddByteArrayOp_Int
13
14 extern StgWord hs_atomic_add8(StgWord x, StgWord val);
15 StgWord
16 hs_atomic_add8(StgWord x, StgWord val)
17 {
18 return __sync_fetch_and_add((volatile StgWord8 *) x, (StgWord8) val);
19 }
20
21 extern StgWord hs_atomic_add16(StgWord x, StgWord val);
22 StgWord
23 hs_atomic_add16(StgWord x, StgWord val)
24 {
25 return __sync_fetch_and_add((volatile StgWord16 *) x, (StgWord16) val);
26 }
27
28 extern StgWord hs_atomic_add32(StgWord x, StgWord val);
29 StgWord
30 hs_atomic_add32(StgWord x, StgWord val)
31 {
32 return __sync_fetch_and_add((volatile StgWord32 *) x, (StgWord32) val);
33 }
34
35 #if WORD_SIZE_IN_BITS == 64
36 extern StgWord64 hs_atomic_add64(StgWord x, StgWord64 val);
37 StgWord64
38 hs_atomic_add64(StgWord x, StgWord64 val)
39 {
40 return __sync_fetch_and_add((volatile StgWord64 *) x, val);
41 }
42 #endif
43
44 // FetchSubByteArrayOp_Int
45
46 extern StgWord hs_atomic_sub8(StgWord x, StgWord val);
47 StgWord
48 hs_atomic_sub8(StgWord x, StgWord val)
49 {
50 return __sync_fetch_and_sub((volatile StgWord8 *) x, (StgWord8) val);
51 }
52
53 extern StgWord hs_atomic_sub16(StgWord x, StgWord val);
54 StgWord
55 hs_atomic_sub16(StgWord x, StgWord val)
56 {
57 return __sync_fetch_and_sub((volatile StgWord16 *) x, (StgWord16) val);
58 }
59
60 extern StgWord hs_atomic_sub32(StgWord x, StgWord val);
61 StgWord
62 hs_atomic_sub32(StgWord x, StgWord val)
63 {
64 return __sync_fetch_and_sub((volatile StgWord32 *) x, (StgWord32) val);
65 }
66
67 #if WORD_SIZE_IN_BITS == 64
68 extern StgWord64 hs_atomic_sub64(StgWord x, StgWord64 val);
69 StgWord64
70 hs_atomic_sub64(StgWord x, StgWord64 val)
71 {
72 return __sync_fetch_and_sub((volatile StgWord64 *) x, val);
73 }
74 #endif
75
76 // FetchAndByteArrayOp_Int
77
78 extern StgWord hs_atomic_and8(StgWord x, StgWord val);
79 StgWord
80 hs_atomic_and8(StgWord x, StgWord val)
81 {
82 return __sync_fetch_and_and((volatile StgWord8 *) x, (StgWord8) val);
83 }
84
85 extern StgWord hs_atomic_and16(StgWord x, StgWord val);
86 StgWord
87 hs_atomic_and16(StgWord x, StgWord val)
88 {
89 return __sync_fetch_and_and((volatile StgWord16 *) x, (StgWord16) val);
90 }
91
92 extern StgWord hs_atomic_and32(StgWord x, StgWord val);
93 StgWord
94 hs_atomic_and32(StgWord x, StgWord val)
95 {
96 return __sync_fetch_and_and((volatile StgWord32 *) x, (StgWord32) val);
97 }
98
99 #if WORD_SIZE_IN_BITS == 64
100 extern StgWord64 hs_atomic_and64(StgWord x, StgWord64 val);
101 StgWord64
102 hs_atomic_and64(StgWord x, StgWord64 val)
103 {
104 return __sync_fetch_and_and((volatile StgWord64 *) x, val);
105 }
106 #endif
107
108 // FetchNandByteArrayOp_Int
109
110 // Workaround for http://llvm.org/bugs/show_bug.cgi?id=8842
111 #define CAS_NAND(x, val) \
112 { \
113 __typeof__ (*(x)) tmp = *(x); \
114 while (!__sync_bool_compare_and_swap(x, tmp, ~(tmp & (val)))) { \
115 tmp = *(x); \
116 } \
117 return tmp; \
118 }
119
120 // This is only provided by clang
121 #if !defined(__has_builtin)
122 #define __has_builtin(x) 0
123 #endif
124
125 // Otherwise this fails with -Werror
126 #pragma GCC diagnostic push
127 #pragma GCC diagnostic ignored "-Wsync-nand"
128
129 extern StgWord hs_atomic_nand8(StgWord x, StgWord val);
130 StgWord
131 hs_atomic_nand8(StgWord x, StgWord val)
132 {
133 #if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
134 CAS_NAND((volatile StgWord8 *) x, (StgWord8) val)
135 #else
136 return __sync_fetch_and_nand((volatile StgWord8 *) x, (StgWord8) val);
137 #endif
138 }
139
140 extern StgWord hs_atomic_nand16(StgWord x, StgWord val);
141 StgWord
142 hs_atomic_nand16(StgWord x, StgWord val)
143 {
144 #if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
145 CAS_NAND((volatile StgWord16 *) x, (StgWord16) val);
146 #else
147 return __sync_fetch_and_nand((volatile StgWord16 *) x, (StgWord16) val);
148 #endif
149 }
150
151 extern StgWord hs_atomic_nand32(StgWord x, StgWord val);
152 StgWord
153 hs_atomic_nand32(StgWord x, StgWord val)
154 {
155 #if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
156 CAS_NAND((volatile StgWord32 *) x, (StgWord32) val);
157 #else
158 return __sync_fetch_and_nand((volatile StgWord32 *) x, (StgWord32) val);
159 #endif
160 }
161
162 #if WORD_SIZE_IN_BITS == 64
163 extern StgWord64 hs_atomic_nand64(StgWord x, StgWord64 val);
164 StgWord64
165 hs_atomic_nand64(StgWord x, StgWord64 val)
166 {
167 #if defined(__clang__) && __has_builtin(__sync_fetch_and_nand)
168 CAS_NAND((volatile StgWord64 *) x, val);
169 #else
170 return __sync_fetch_and_nand((volatile StgWord64 *) x, val);
171 #endif
172 }
173 #endif
174
175 #pragma GCC diagnostic pop
176
177 // FetchOrByteArrayOp_Int
178
179 extern StgWord hs_atomic_or8(StgWord x, StgWord val);
180 StgWord
181 hs_atomic_or8(StgWord x, StgWord val)
182 {
183 return __sync_fetch_and_or((volatile StgWord8 *) x, (StgWord8) val);
184 }
185
186 extern StgWord hs_atomic_or16(StgWord x, StgWord val);
187 StgWord
188 hs_atomic_or16(StgWord x, StgWord val)
189 {
190 return __sync_fetch_and_or((volatile StgWord16 *) x, (StgWord16) val);
191 }
192
193 extern StgWord hs_atomic_or32(StgWord x, StgWord val);
194 StgWord
195 hs_atomic_or32(StgWord x, StgWord val)
196 {
197 return __sync_fetch_and_or((volatile StgWord32 *) x, (StgWord32) val);
198 }
199
200 #if WORD_SIZE_IN_BITS == 64
201 extern StgWord64 hs_atomic_or64(StgWord x, StgWord64 val);
202 StgWord64
203 hs_atomic_or64(StgWord x, StgWord64 val)
204 {
205 return __sync_fetch_and_or((volatile StgWord64 *) x, val);
206 }
207 #endif
208
209 // FetchXorByteArrayOp_Int
210
211 extern StgWord hs_atomic_xor8(StgWord x, StgWord val);
212 StgWord
213 hs_atomic_xor8(StgWord x, StgWord val)
214 {
215 return __sync_fetch_and_xor((volatile StgWord8 *) x, (StgWord8) val);
216 }
217
218 extern StgWord hs_atomic_xor16(StgWord x, StgWord val);
219 StgWord
220 hs_atomic_xor16(StgWord x, StgWord val)
221 {
222 return __sync_fetch_and_xor((volatile StgWord16 *) x, (StgWord16) val);
223 }
224
225 extern StgWord hs_atomic_xor32(StgWord x, StgWord val);
226 StgWord
227 hs_atomic_xor32(StgWord x, StgWord val)
228 {
229 return __sync_fetch_and_xor((volatile StgWord32 *) x, (StgWord32) val);
230 }
231
232 #if WORD_SIZE_IN_BITS == 64
233 extern StgWord64 hs_atomic_xor64(StgWord x, StgWord64 val);
234 StgWord64
235 hs_atomic_xor64(StgWord x, StgWord64 val)
236 {
237 return __sync_fetch_and_xor((volatile StgWord64 *) x, val);
238 }
239 #endif
240
241 // CasByteArrayOp_Int
242
243 extern StgWord hs_cmpxchg8(StgWord x, StgWord old, StgWord new);
244 StgWord
245 hs_cmpxchg8(StgWord x, StgWord old, StgWord new)
246 {
247 return __sync_val_compare_and_swap((volatile StgWord8 *) x, (StgWord8) old, (StgWord8) new);
248 }
249
250 extern StgWord hs_cmpxchg16(StgWord x, StgWord old, StgWord new);
251 StgWord
252 hs_cmpxchg16(StgWord x, StgWord old, StgWord new)
253 {
254 return __sync_val_compare_and_swap((volatile StgWord16 *) x, (StgWord16) old, (StgWord16) new);
255 }
256
257 extern StgWord hs_cmpxchg32(StgWord x, StgWord old, StgWord new);
258 StgWord
259 hs_cmpxchg32(StgWord x, StgWord old, StgWord new)
260 {
261 return __sync_val_compare_and_swap((volatile StgWord32 *) x, (StgWord32) old, (StgWord32) new);
262 }
263
264 #if WORD_SIZE_IN_BITS == 64
265 extern StgWord hs_cmpxchg64(StgWord x, StgWord64 old, StgWord64 new);
266 StgWord
267 hs_cmpxchg64(StgWord x, StgWord64 old, StgWord64 new)
268 {
269 return __sync_val_compare_and_swap((volatile StgWord64 *) x, old, new);
270 }
271 #endif
272
273 // AtomicReadByteArrayOp_Int
274 // Implies a full memory barrier (see compiler/prelude/primops.txt.pp)
275 // __ATOMIC_SEQ_CST: Full barrier in both directions (hoisting and sinking
276 // of code) and synchronizes with acquire loads and release stores in
277 // all threads.
278 //
279 // When we lack C11 atomics support we emulate these using the old GCC __sync
280 // primitives which the GCC documentation claims "usually" implies a full
281 // barrier.
282
283 extern StgWord hs_atomicread8(StgWord x);
284 StgWord
285 hs_atomicread8(StgWord x)
286 {
287 #if HAVE_C11_ATOMICS
288 return __atomic_load_n((StgWord8 *) x, __ATOMIC_SEQ_CST);
289 #else
290 return __sync_add_and_fetch((StgWord8 *) x, 0);
291 #endif
292 }
293
294 extern StgWord hs_atomicread16(StgWord x);
295 StgWord
296 hs_atomicread16(StgWord x)
297 {
298 #if HAVE_C11_ATOMICS
299 return __atomic_load_n((StgWord16 *) x, __ATOMIC_SEQ_CST);
300 #else
301 return __sync_add_and_fetch((StgWord16 *) x, 0);
302 #endif
303 }
304
305 extern StgWord hs_atomicread32(StgWord x);
306 StgWord
307 hs_atomicread32(StgWord x)
308 {
309 #if HAVE_C11_ATOMICS
310 return __atomic_load_n((StgWord32 *) x, __ATOMIC_SEQ_CST);
311 #else
312 return __sync_add_and_fetch((StgWord32 *) x, 0);
313 #endif
314 }
315
316 extern StgWord64 hs_atomicread64(StgWord x);
317 StgWord64
318 hs_atomicread64(StgWord x)
319 {
320 #if HAVE_C11_ATOMICS
321 return __atomic_load_n((StgWord64 *) x, __ATOMIC_SEQ_CST);
322 #else
323 return __sync_add_and_fetch((StgWord64 *) x, 0);
324 #endif
325 }
326
327 // AtomicWriteByteArrayOp_Int
328 // Implies a full memory barrier (see compiler/prelude/primops.txt.pp)
329 // __ATOMIC_SEQ_CST: Full barrier (see hs_atomicread8 above).
330
331 extern void hs_atomicwrite8(StgWord x, StgWord val);
332 void
333 hs_atomicwrite8(StgWord x, StgWord val)
334 {
335 #if HAVE_C11_ATOMICS
336 __atomic_store_n((StgWord8 *) x, (StgWord8) val, __ATOMIC_SEQ_CST);
337 #else
338 while (!__sync_bool_compare_and_swap((StgWord8 *) x, *(StgWord8 *) x, (StgWord8) val));
339 #endif
340 }
341
342 extern void hs_atomicwrite16(StgWord x, StgWord val);
343 void
344 hs_atomicwrite16(StgWord x, StgWord val)
345 {
346 #if HAVE_C11_ATOMICS
347 __atomic_store_n((StgWord16 *) x, (StgWord16) val, __ATOMIC_SEQ_CST);
348 #else
349 while (!__sync_bool_compare_and_swap((StgWord16 *) x, *(StgWord16 *) x, (StgWord16) val));
350 #endif
351 }
352
353 extern void hs_atomicwrite32(StgWord x, StgWord val);
354 void
355 hs_atomicwrite32(StgWord x, StgWord val)
356 {
357 #if HAVE_C11_ATOMICS
358 __atomic_store_n((StgWord32 *) x, (StgWord32) val, __ATOMIC_SEQ_CST);
359 #else
360 while (!__sync_bool_compare_and_swap((StgWord32 *) x, *(StgWord32 *) x, (StgWord32) val));
361 #endif
362 }
363
364 extern void hs_atomicwrite64(StgWord x, StgWord64 val);
365 void
366 hs_atomicwrite64(StgWord x, StgWord64 val)
367 {
368 #if HAVE_C11_ATOMICS
369 __atomic_store_n((StgWord64 *) x, (StgWord64) val, __ATOMIC_SEQ_CST);
370 #else
371 while (!__sync_bool_compare_and_swap((StgWord64 *) x, *(StgWord64 *) x, (StgWord64) val));
372 #endif
373 }