1*4882a593Smuzhiyun// SPDX-License-Identifier: GPL-2.0+ 2*4882a593Smuzhiyun// 3*4882a593Smuzhiyun// An earlier version of this file appeared in the companion webpage for 4*4882a593Smuzhiyun// "Frightening small children and disconcerting grown-ups: Concurrency 5*4882a593Smuzhiyun// in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern, 6*4882a593Smuzhiyun// which appeared in ASPLOS 2018. 7*4882a593Smuzhiyun 8*4882a593Smuzhiyun// ONCE 9*4882a593SmuzhiyunREAD_ONCE(X) __load{once}(X) 10*4882a593SmuzhiyunWRITE_ONCE(X,V) { __store{once}(X,V); } 11*4882a593Smuzhiyun 12*4882a593Smuzhiyun// Release Acquire and friends 13*4882a593Smuzhiyunsmp_store_release(X,V) { __store{release}(*X,V); } 14*4882a593Smuzhiyunsmp_load_acquire(X) __load{acquire}(*X) 15*4882a593Smuzhiyunrcu_assign_pointer(X,V) { __store{release}(X,V); } 16*4882a593Smuzhiyunrcu_dereference(X) __load{once}(X) 17*4882a593Smuzhiyunsmp_store_mb(X,V) { __store{once}(X,V); __fence{mb}; } 18*4882a593Smuzhiyun 19*4882a593Smuzhiyun// Fences 20*4882a593Smuzhiyunsmp_mb() { __fence{mb}; } 21*4882a593Smuzhiyunsmp_rmb() { __fence{rmb}; } 22*4882a593Smuzhiyunsmp_wmb() { __fence{wmb}; } 23*4882a593Smuzhiyunsmp_mb__before_atomic() { __fence{before-atomic}; } 24*4882a593Smuzhiyunsmp_mb__after_atomic() { __fence{after-atomic}; } 25*4882a593Smuzhiyunsmp_mb__after_spinlock() { __fence{after-spinlock}; } 26*4882a593Smuzhiyunsmp_mb__after_unlock_lock() { __fence{after-unlock-lock}; } 27*4882a593Smuzhiyunbarrier() { __fence{barrier}; } 28*4882a593Smuzhiyun 29*4882a593Smuzhiyun// Exchange 30*4882a593Smuzhiyunxchg(X,V) __xchg{mb}(X,V) 31*4882a593Smuzhiyunxchg_relaxed(X,V) __xchg{once}(X,V) 32*4882a593Smuzhiyunxchg_release(X,V) __xchg{release}(X,V) 33*4882a593Smuzhiyunxchg_acquire(X,V) __xchg{acquire}(X,V) 34*4882a593Smuzhiyuncmpxchg(X,V,W) __cmpxchg{mb}(X,V,W) 35*4882a593Smuzhiyuncmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W) 36*4882a593Smuzhiyuncmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W) 37*4882a593Smuzhiyuncmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W) 38*4882a593Smuzhiyun 39*4882a593Smuzhiyun// Spinlocks 40*4882a593Smuzhiyunspin_lock(X) { __lock(X); } 41*4882a593Smuzhiyunspin_unlock(X) { __unlock(X); } 42*4882a593Smuzhiyunspin_trylock(X) __trylock(X) 43*4882a593Smuzhiyunspin_is_locked(X) __islocked(X) 44*4882a593Smuzhiyun 45*4882a593Smuzhiyun// RCU 46*4882a593Smuzhiyunrcu_read_lock() { __fence{rcu-lock}; } 47*4882a593Smuzhiyunrcu_read_unlock() { __fence{rcu-unlock}; } 48*4882a593Smuzhiyunsynchronize_rcu() { __fence{sync-rcu}; } 49*4882a593Smuzhiyunsynchronize_rcu_expedited() { __fence{sync-rcu}; } 50*4882a593Smuzhiyun 51*4882a593Smuzhiyun// SRCU 52*4882a593Smuzhiyunsrcu_read_lock(X) __srcu{srcu-lock}(X) 53*4882a593Smuzhiyunsrcu_read_unlock(X,Y) { __srcu{srcu-unlock}(X,Y); } 54*4882a593Smuzhiyunsynchronize_srcu(X) { __srcu{sync-srcu}(X); } 55*4882a593Smuzhiyunsynchronize_srcu_expedited(X) { __srcu{sync-srcu}(X); } 56*4882a593Smuzhiyun 57*4882a593Smuzhiyun// Atomic 58*4882a593Smuzhiyunatomic_read(X) READ_ONCE(*X) 59*4882a593Smuzhiyunatomic_set(X,V) { WRITE_ONCE(*X,V); } 60*4882a593Smuzhiyunatomic_read_acquire(X) smp_load_acquire(X) 61*4882a593Smuzhiyunatomic_set_release(X,V) { smp_store_release(X,V); } 62*4882a593Smuzhiyun 63*4882a593Smuzhiyunatomic_add(V,X) { __atomic_op(X,+,V); } 64*4882a593Smuzhiyunatomic_sub(V,X) { __atomic_op(X,-,V); } 65*4882a593Smuzhiyunatomic_inc(X) { __atomic_op(X,+,1); } 66*4882a593Smuzhiyunatomic_dec(X) { __atomic_op(X,-,1); } 67*4882a593Smuzhiyun 68*4882a593Smuzhiyunatomic_add_return(V,X) __atomic_op_return{mb}(X,+,V) 69*4882a593Smuzhiyunatomic_add_return_relaxed(V,X) __atomic_op_return{once}(X,+,V) 70*4882a593Smuzhiyunatomic_add_return_acquire(V,X) __atomic_op_return{acquire}(X,+,V) 71*4882a593Smuzhiyunatomic_add_return_release(V,X) __atomic_op_return{release}(X,+,V) 72*4882a593Smuzhiyunatomic_fetch_add(V,X) __atomic_fetch_op{mb}(X,+,V) 73*4882a593Smuzhiyunatomic_fetch_add_relaxed(V,X) __atomic_fetch_op{once}(X,+,V) 74*4882a593Smuzhiyunatomic_fetch_add_acquire(V,X) __atomic_fetch_op{acquire}(X,+,V) 75*4882a593Smuzhiyunatomic_fetch_add_release(V,X) __atomic_fetch_op{release}(X,+,V) 76*4882a593Smuzhiyun 77*4882a593Smuzhiyunatomic_inc_return(X) __atomic_op_return{mb}(X,+,1) 78*4882a593Smuzhiyunatomic_inc_return_relaxed(X) __atomic_op_return{once}(X,+,1) 79*4882a593Smuzhiyunatomic_inc_return_acquire(X) __atomic_op_return{acquire}(X,+,1) 80*4882a593Smuzhiyunatomic_inc_return_release(X) __atomic_op_return{release}(X,+,1) 81*4882a593Smuzhiyunatomic_fetch_inc(X) __atomic_fetch_op{mb}(X,+,1) 82*4882a593Smuzhiyunatomic_fetch_inc_relaxed(X) __atomic_fetch_op{once}(X,+,1) 83*4882a593Smuzhiyunatomic_fetch_inc_acquire(X) __atomic_fetch_op{acquire}(X,+,1) 84*4882a593Smuzhiyunatomic_fetch_inc_release(X) __atomic_fetch_op{release}(X,+,1) 85*4882a593Smuzhiyun 86*4882a593Smuzhiyunatomic_sub_return(V,X) __atomic_op_return{mb}(X,-,V) 87*4882a593Smuzhiyunatomic_sub_return_relaxed(V,X) __atomic_op_return{once}(X,-,V) 88*4882a593Smuzhiyunatomic_sub_return_acquire(V,X) __atomic_op_return{acquire}(X,-,V) 89*4882a593Smuzhiyunatomic_sub_return_release(V,X) __atomic_op_return{release}(X,-,V) 90*4882a593Smuzhiyunatomic_fetch_sub(V,X) __atomic_fetch_op{mb}(X,-,V) 91*4882a593Smuzhiyunatomic_fetch_sub_relaxed(V,X) __atomic_fetch_op{once}(X,-,V) 92*4882a593Smuzhiyunatomic_fetch_sub_acquire(V,X) __atomic_fetch_op{acquire}(X,-,V) 93*4882a593Smuzhiyunatomic_fetch_sub_release(V,X) __atomic_fetch_op{release}(X,-,V) 94*4882a593Smuzhiyun 95*4882a593Smuzhiyunatomic_dec_return(X) __atomic_op_return{mb}(X,-,1) 96*4882a593Smuzhiyunatomic_dec_return_relaxed(X) __atomic_op_return{once}(X,-,1) 97*4882a593Smuzhiyunatomic_dec_return_acquire(X) __atomic_op_return{acquire}(X,-,1) 98*4882a593Smuzhiyunatomic_dec_return_release(X) __atomic_op_return{release}(X,-,1) 99*4882a593Smuzhiyunatomic_fetch_dec(X) __atomic_fetch_op{mb}(X,-,1) 100*4882a593Smuzhiyunatomic_fetch_dec_relaxed(X) __atomic_fetch_op{once}(X,-,1) 101*4882a593Smuzhiyunatomic_fetch_dec_acquire(X) __atomic_fetch_op{acquire}(X,-,1) 102*4882a593Smuzhiyunatomic_fetch_dec_release(X) __atomic_fetch_op{release}(X,-,1) 103*4882a593Smuzhiyun 104*4882a593Smuzhiyunatomic_xchg(X,V) __xchg{mb}(X,V) 105*4882a593Smuzhiyunatomic_xchg_relaxed(X,V) __xchg{once}(X,V) 106*4882a593Smuzhiyunatomic_xchg_release(X,V) __xchg{release}(X,V) 107*4882a593Smuzhiyunatomic_xchg_acquire(X,V) __xchg{acquire}(X,V) 108*4882a593Smuzhiyunatomic_cmpxchg(X,V,W) __cmpxchg{mb}(X,V,W) 109*4882a593Smuzhiyunatomic_cmpxchg_relaxed(X,V,W) __cmpxchg{once}(X,V,W) 110*4882a593Smuzhiyunatomic_cmpxchg_acquire(X,V,W) __cmpxchg{acquire}(X,V,W) 111*4882a593Smuzhiyunatomic_cmpxchg_release(X,V,W) __cmpxchg{release}(X,V,W) 112*4882a593Smuzhiyun 113*4882a593Smuzhiyunatomic_sub_and_test(V,X) __atomic_op_return{mb}(X,-,V) == 0 114*4882a593Smuzhiyunatomic_dec_and_test(X) __atomic_op_return{mb}(X,-,1) == 0 115*4882a593Smuzhiyunatomic_inc_and_test(X) __atomic_op_return{mb}(X,+,1) == 0 116*4882a593Smuzhiyunatomic_add_negative(V,X) __atomic_op_return{mb}(X,+,V) < 0 117