1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0-only */ 2*4882a593Smuzhiyun /* 3*4882a593Smuzhiyun * Based on arch/arm/include/asm/barrier.h 4*4882a593Smuzhiyun * 5*4882a593Smuzhiyun * Copyright (C) 2012 ARM Ltd. 6*4882a593Smuzhiyun * Copyright (C) 2013 Regents of the University of California 7*4882a593Smuzhiyun * Copyright (C) 2017 SiFive 8*4882a593Smuzhiyun */ 9*4882a593Smuzhiyun 10*4882a593Smuzhiyun #ifndef _ASM_RISCV_BARRIER_H 11*4882a593Smuzhiyun #define _ASM_RISCV_BARRIER_H 12*4882a593Smuzhiyun 13*4882a593Smuzhiyun #ifndef __ASSEMBLY__ 14*4882a593Smuzhiyun 15*4882a593Smuzhiyun #define nop() __asm__ __volatile__ ("nop") 16*4882a593Smuzhiyun 17*4882a593Smuzhiyun #define RISCV_FENCE(p, s) \ 18*4882a593Smuzhiyun __asm__ __volatile__ ("fence " #p "," #s : : : "memory") 19*4882a593Smuzhiyun 20*4882a593Smuzhiyun /* These barriers need to enforce ordering on both devices or memory. */ 21*4882a593Smuzhiyun #define mb() RISCV_FENCE(iorw,iorw) 22*4882a593Smuzhiyun #define rmb() RISCV_FENCE(ir,ir) 23*4882a593Smuzhiyun #define wmb() RISCV_FENCE(ow,ow) 24*4882a593Smuzhiyun 25*4882a593Smuzhiyun /* These barriers do not need to enforce ordering on devices, just memory. */ 26*4882a593Smuzhiyun #define __smp_mb() RISCV_FENCE(rw,rw) 27*4882a593Smuzhiyun #define __smp_rmb() RISCV_FENCE(r,r) 28*4882a593Smuzhiyun #define __smp_wmb() RISCV_FENCE(w,w) 29*4882a593Smuzhiyun 30*4882a593Smuzhiyun #define __smp_store_release(p, v) \ 31*4882a593Smuzhiyun do { \ 32*4882a593Smuzhiyun compiletime_assert_atomic_type(*p); \ 33*4882a593Smuzhiyun RISCV_FENCE(rw,w); \ 34*4882a593Smuzhiyun WRITE_ONCE(*p, v); \ 35*4882a593Smuzhiyun } while (0) 36*4882a593Smuzhiyun 37*4882a593Smuzhiyun #define __smp_load_acquire(p) \ 38*4882a593Smuzhiyun ({ \ 39*4882a593Smuzhiyun typeof(*p) ___p1 = READ_ONCE(*p); \ 40*4882a593Smuzhiyun compiletime_assert_atomic_type(*p); \ 41*4882a593Smuzhiyun RISCV_FENCE(r,rw); \ 42*4882a593Smuzhiyun ___p1; \ 43*4882a593Smuzhiyun }) 44*4882a593Smuzhiyun 45*4882a593Smuzhiyun /* 46*4882a593Smuzhiyun * This is a very specific barrier: it's currently only used in two places in 47*4882a593Smuzhiyun * the kernel, both in the scheduler. See include/linux/spinlock.h for the two 48*4882a593Smuzhiyun * orderings it guarantees, but the "critical section is RCsc" guarantee 49*4882a593Smuzhiyun * mandates a barrier on RISC-V. The sequence looks like: 50*4882a593Smuzhiyun * 51*4882a593Smuzhiyun * lr.aq lock 52*4882a593Smuzhiyun * sc lock <= LOCKED 53*4882a593Smuzhiyun * smp_mb__after_spinlock() 54*4882a593Smuzhiyun * // critical section 55*4882a593Smuzhiyun * lr lock 56*4882a593Smuzhiyun * sc.rl lock <= UNLOCKED 57*4882a593Smuzhiyun * 58*4882a593Smuzhiyun * The AQ/RL pair provides a RCpc critical section, but there's not really any 59*4882a593Smuzhiyun * way we can take advantage of that here because the ordering is only enforced 60*4882a593Smuzhiyun * on that one lock. Thus, we're just doing a full fence. 61*4882a593Smuzhiyun * 62*4882a593Smuzhiyun * Since we allow writeX to be called from preemptive regions we need at least 63*4882a593Smuzhiyun * an "o" in the predecessor set to ensure device writes are visible before the 64*4882a593Smuzhiyun * task is marked as available for scheduling on a new hart. While I don't see 65*4882a593Smuzhiyun * any concrete reason we need a full IO fence, it seems safer to just upgrade 66*4882a593Smuzhiyun * this in order to avoid any IO crossing a scheduling boundary. In both 67*4882a593Smuzhiyun * instances the scheduler pairs this with an mb(), so nothing is necessary on 68*4882a593Smuzhiyun * the new hart. 69*4882a593Smuzhiyun */ 70*4882a593Smuzhiyun #define smp_mb__after_spinlock() RISCV_FENCE(iorw,iorw) 71*4882a593Smuzhiyun 72*4882a593Smuzhiyun #include <asm-generic/barrier.h> 73*4882a593Smuzhiyun 74*4882a593Smuzhiyun #endif /* __ASSEMBLY__ */ 75*4882a593Smuzhiyun 76*4882a593Smuzhiyun #endif /* _ASM_RISCV_BARRIER_H */ 77