1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0 */
2*4882a593Smuzhiyun #ifndef _ASM_X86_BARRIER_H
3*4882a593Smuzhiyun #define _ASM_X86_BARRIER_H
4*4882a593Smuzhiyun
5*4882a593Smuzhiyun #include <asm/alternative.h>
6*4882a593Smuzhiyun #include <asm/nops.h>
7*4882a593Smuzhiyun
8*4882a593Smuzhiyun /*
9*4882a593Smuzhiyun * Force strict CPU ordering.
10*4882a593Smuzhiyun * And yes, this might be required on UP too when we're talking
11*4882a593Smuzhiyun * to devices.
12*4882a593Smuzhiyun */
13*4882a593Smuzhiyun
14*4882a593Smuzhiyun #ifdef CONFIG_X86_32
15*4882a593Smuzhiyun #define mb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "mfence", \
16*4882a593Smuzhiyun X86_FEATURE_XMM2) ::: "memory", "cc")
17*4882a593Smuzhiyun #define rmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "lfence", \
18*4882a593Smuzhiyun X86_FEATURE_XMM2) ::: "memory", "cc")
19*4882a593Smuzhiyun #define wmb() asm volatile(ALTERNATIVE("lock; addl $0,-4(%%esp)", "sfence", \
20*4882a593Smuzhiyun X86_FEATURE_XMM2) ::: "memory", "cc")
21*4882a593Smuzhiyun #else
22*4882a593Smuzhiyun #define mb() asm volatile("mfence":::"memory")
23*4882a593Smuzhiyun #define rmb() asm volatile("lfence":::"memory")
24*4882a593Smuzhiyun #define wmb() asm volatile("sfence" ::: "memory")
25*4882a593Smuzhiyun #endif
26*4882a593Smuzhiyun
27*4882a593Smuzhiyun /**
28*4882a593Smuzhiyun * array_index_mask_nospec() - generate a mask that is ~0UL when the
29*4882a593Smuzhiyun * bounds check succeeds and 0 otherwise
30*4882a593Smuzhiyun * @index: array element index
31*4882a593Smuzhiyun * @size: number of elements in array
32*4882a593Smuzhiyun *
33*4882a593Smuzhiyun * Returns:
34*4882a593Smuzhiyun * 0 - (index < size)
35*4882a593Smuzhiyun */
array_index_mask_nospec(unsigned long index,unsigned long size)36*4882a593Smuzhiyun static inline unsigned long array_index_mask_nospec(unsigned long index,
37*4882a593Smuzhiyun unsigned long size)
38*4882a593Smuzhiyun {
39*4882a593Smuzhiyun unsigned long mask;
40*4882a593Smuzhiyun
41*4882a593Smuzhiyun asm volatile ("cmp %1,%2; sbb %0,%0;"
42*4882a593Smuzhiyun :"=r" (mask)
43*4882a593Smuzhiyun :"g"(size),"r" (index)
44*4882a593Smuzhiyun :"cc");
45*4882a593Smuzhiyun return mask;
46*4882a593Smuzhiyun }
47*4882a593Smuzhiyun
48*4882a593Smuzhiyun /* Override the default implementation from linux/nospec.h. */
49*4882a593Smuzhiyun #define array_index_mask_nospec array_index_mask_nospec
50*4882a593Smuzhiyun
51*4882a593Smuzhiyun /* Prevent speculative execution past this barrier. */
52*4882a593Smuzhiyun #define barrier_nospec() alternative("", "lfence", X86_FEATURE_LFENCE_RDTSC)
53*4882a593Smuzhiyun
54*4882a593Smuzhiyun #define dma_rmb() barrier()
55*4882a593Smuzhiyun #define dma_wmb() barrier()
56*4882a593Smuzhiyun
57*4882a593Smuzhiyun #ifdef CONFIG_X86_32
58*4882a593Smuzhiyun #define __smp_mb() asm volatile("lock; addl $0,-4(%%esp)" ::: "memory", "cc")
59*4882a593Smuzhiyun #else
60*4882a593Smuzhiyun #define __smp_mb() asm volatile("lock; addl $0,-4(%%rsp)" ::: "memory", "cc")
61*4882a593Smuzhiyun #endif
62*4882a593Smuzhiyun #define __smp_rmb() dma_rmb()
63*4882a593Smuzhiyun #define __smp_wmb() barrier()
64*4882a593Smuzhiyun #define __smp_store_mb(var, value) do { (void)xchg(&var, value); } while (0)
65*4882a593Smuzhiyun
66*4882a593Smuzhiyun #define __smp_store_release(p, v) \
67*4882a593Smuzhiyun do { \
68*4882a593Smuzhiyun compiletime_assert_atomic_type(*p); \
69*4882a593Smuzhiyun barrier(); \
70*4882a593Smuzhiyun WRITE_ONCE(*p, v); \
71*4882a593Smuzhiyun } while (0)
72*4882a593Smuzhiyun
73*4882a593Smuzhiyun #define __smp_load_acquire(p) \
74*4882a593Smuzhiyun ({ \
75*4882a593Smuzhiyun typeof(*p) ___p1 = READ_ONCE(*p); \
76*4882a593Smuzhiyun compiletime_assert_atomic_type(*p); \
77*4882a593Smuzhiyun barrier(); \
78*4882a593Smuzhiyun ___p1; \
79*4882a593Smuzhiyun })
80*4882a593Smuzhiyun
81*4882a593Smuzhiyun /* Atomic operations are already serializing on x86 */
82*4882a593Smuzhiyun #define __smp_mb__before_atomic() do { } while (0)
83*4882a593Smuzhiyun #define __smp_mb__after_atomic() do { } while (0)
84*4882a593Smuzhiyun
85*4882a593Smuzhiyun #include <asm-generic/barrier.h>
86*4882a593Smuzhiyun
87*4882a593Smuzhiyun /*
88*4882a593Smuzhiyun * Make previous memory operations globally visible before
89*4882a593Smuzhiyun * a WRMSR.
90*4882a593Smuzhiyun *
91*4882a593Smuzhiyun * MFENCE makes writes visible, but only affects load/store
92*4882a593Smuzhiyun * instructions. WRMSR is unfortunately not a load/store
93*4882a593Smuzhiyun * instruction and is unaffected by MFENCE. The LFENCE ensures
94*4882a593Smuzhiyun * that the WRMSR is not reordered.
95*4882a593Smuzhiyun *
96*4882a593Smuzhiyun * Most WRMSRs are full serializing instructions themselves and
97*4882a593Smuzhiyun * do not require this barrier. This is only required for the
98*4882a593Smuzhiyun * IA32_TSC_DEADLINE and X2APIC MSRs.
99*4882a593Smuzhiyun */
weak_wrmsr_fence(void)100*4882a593Smuzhiyun static inline void weak_wrmsr_fence(void)
101*4882a593Smuzhiyun {
102*4882a593Smuzhiyun asm volatile("mfence; lfence" : : : "memory");
103*4882a593Smuzhiyun }
104*4882a593Smuzhiyun
105*4882a593Smuzhiyun #endif /* _ASM_X86_BARRIER_H */
106