1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0 */ 2*4882a593Smuzhiyun #ifndef __VDSO_HELPERS_H 3*4882a593Smuzhiyun #define __VDSO_HELPERS_H 4*4882a593Smuzhiyun 5*4882a593Smuzhiyun #ifndef __ASSEMBLY__ 6*4882a593Smuzhiyun 7*4882a593Smuzhiyun #include <vdso/datapage.h> 8*4882a593Smuzhiyun vdso_read_begin(const struct vdso_data * vd)9*4882a593Smuzhiyunstatic __always_inline u32 vdso_read_begin(const struct vdso_data *vd) 10*4882a593Smuzhiyun { 11*4882a593Smuzhiyun u32 seq; 12*4882a593Smuzhiyun 13*4882a593Smuzhiyun while (unlikely((seq = READ_ONCE(vd->seq)) & 1)) 14*4882a593Smuzhiyun cpu_relax(); 15*4882a593Smuzhiyun 16*4882a593Smuzhiyun smp_rmb(); 17*4882a593Smuzhiyun return seq; 18*4882a593Smuzhiyun } 19*4882a593Smuzhiyun vdso_read_retry(const struct vdso_data * vd,u32 start)20*4882a593Smuzhiyunstatic __always_inline u32 vdso_read_retry(const struct vdso_data *vd, 21*4882a593Smuzhiyun u32 start) 22*4882a593Smuzhiyun { 23*4882a593Smuzhiyun u32 seq; 24*4882a593Smuzhiyun 25*4882a593Smuzhiyun smp_rmb(); 26*4882a593Smuzhiyun seq = READ_ONCE(vd->seq); 27*4882a593Smuzhiyun return seq != start; 28*4882a593Smuzhiyun } 29*4882a593Smuzhiyun vdso_write_begin(struct vdso_data * vd)30*4882a593Smuzhiyunstatic __always_inline void vdso_write_begin(struct vdso_data *vd) 31*4882a593Smuzhiyun { 32*4882a593Smuzhiyun /* 33*4882a593Smuzhiyun * WRITE_ONCE it is required otherwise the compiler can validly tear 34*4882a593Smuzhiyun * updates to vd[x].seq and it is possible that the value seen by the 35*4882a593Smuzhiyun * reader it is inconsistent. 36*4882a593Smuzhiyun */ 37*4882a593Smuzhiyun WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1); 38*4882a593Smuzhiyun WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1); 39*4882a593Smuzhiyun smp_wmb(); 40*4882a593Smuzhiyun } 41*4882a593Smuzhiyun vdso_write_end(struct vdso_data * vd)42*4882a593Smuzhiyunstatic __always_inline void vdso_write_end(struct vdso_data *vd) 43*4882a593Smuzhiyun { 44*4882a593Smuzhiyun smp_wmb(); 45*4882a593Smuzhiyun /* 46*4882a593Smuzhiyun * WRITE_ONCE it is required otherwise the compiler can validly tear 47*4882a593Smuzhiyun * updates to vd[x].seq and it is possible that the value seen by the 48*4882a593Smuzhiyun * reader it is inconsistent. 49*4882a593Smuzhiyun */ 50*4882a593Smuzhiyun WRITE_ONCE(vd[CS_HRES_COARSE].seq, vd[CS_HRES_COARSE].seq + 1); 51*4882a593Smuzhiyun WRITE_ONCE(vd[CS_RAW].seq, vd[CS_RAW].seq + 1); 52*4882a593Smuzhiyun } 53*4882a593Smuzhiyun 54*4882a593Smuzhiyun #endif /* !__ASSEMBLY__ */ 55*4882a593Smuzhiyun 56*4882a593Smuzhiyun #endif /* __VDSO_HELPERS_H */ 57