1*4882a593Smuzhiyun/* SPDX-License-Identifier: GPL-2.0-or-later */ 2*4882a593Smuzhiyun/* 3*4882a593Smuzhiyun * atomic64_t for 586+ 4*4882a593Smuzhiyun * 5*4882a593Smuzhiyun * Copyright © 2010 Luca Barbieri 6*4882a593Smuzhiyun */ 7*4882a593Smuzhiyun 8*4882a593Smuzhiyun#include <linux/linkage.h> 9*4882a593Smuzhiyun#include <asm/alternative.h> 10*4882a593Smuzhiyun 11*4882a593Smuzhiyun.macro read64 reg 12*4882a593Smuzhiyun movl %ebx, %eax 13*4882a593Smuzhiyun movl %ecx, %edx 14*4882a593Smuzhiyun/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */ 15*4882a593Smuzhiyun LOCK_PREFIX 16*4882a593Smuzhiyun cmpxchg8b (\reg) 17*4882a593Smuzhiyun.endm 18*4882a593Smuzhiyun 19*4882a593SmuzhiyunSYM_FUNC_START(atomic64_read_cx8) 20*4882a593Smuzhiyun read64 %ecx 21*4882a593Smuzhiyun RET 22*4882a593SmuzhiyunSYM_FUNC_END(atomic64_read_cx8) 23*4882a593Smuzhiyun 24*4882a593SmuzhiyunSYM_FUNC_START(atomic64_set_cx8) 25*4882a593Smuzhiyun1: 26*4882a593Smuzhiyun/* we don't need LOCK_PREFIX since aligned 64-bit writes 27*4882a593Smuzhiyun * are atomic on 586 and newer */ 28*4882a593Smuzhiyun cmpxchg8b (%esi) 29*4882a593Smuzhiyun jne 1b 30*4882a593Smuzhiyun 31*4882a593Smuzhiyun RET 32*4882a593SmuzhiyunSYM_FUNC_END(atomic64_set_cx8) 33*4882a593Smuzhiyun 34*4882a593SmuzhiyunSYM_FUNC_START(atomic64_xchg_cx8) 35*4882a593Smuzhiyun1: 36*4882a593Smuzhiyun LOCK_PREFIX 37*4882a593Smuzhiyun cmpxchg8b (%esi) 38*4882a593Smuzhiyun jne 1b 39*4882a593Smuzhiyun 40*4882a593Smuzhiyun RET 41*4882a593SmuzhiyunSYM_FUNC_END(atomic64_xchg_cx8) 42*4882a593Smuzhiyun 43*4882a593Smuzhiyun.macro addsub_return func ins insc 44*4882a593SmuzhiyunSYM_FUNC_START(atomic64_\func\()_return_cx8) 45*4882a593Smuzhiyun pushl %ebp 46*4882a593Smuzhiyun pushl %ebx 47*4882a593Smuzhiyun pushl %esi 48*4882a593Smuzhiyun pushl %edi 49*4882a593Smuzhiyun 50*4882a593Smuzhiyun movl %eax, %esi 51*4882a593Smuzhiyun movl %edx, %edi 52*4882a593Smuzhiyun movl %ecx, %ebp 53*4882a593Smuzhiyun 54*4882a593Smuzhiyun read64 %ecx 55*4882a593Smuzhiyun1: 56*4882a593Smuzhiyun movl %eax, %ebx 57*4882a593Smuzhiyun movl %edx, %ecx 58*4882a593Smuzhiyun \ins\()l %esi, %ebx 59*4882a593Smuzhiyun \insc\()l %edi, %ecx 60*4882a593Smuzhiyun LOCK_PREFIX 61*4882a593Smuzhiyun cmpxchg8b (%ebp) 62*4882a593Smuzhiyun jne 1b 63*4882a593Smuzhiyun 64*4882a593Smuzhiyun10: 65*4882a593Smuzhiyun movl %ebx, %eax 66*4882a593Smuzhiyun movl %ecx, %edx 67*4882a593Smuzhiyun popl %edi 68*4882a593Smuzhiyun popl %esi 69*4882a593Smuzhiyun popl %ebx 70*4882a593Smuzhiyun popl %ebp 71*4882a593Smuzhiyun RET 72*4882a593SmuzhiyunSYM_FUNC_END(atomic64_\func\()_return_cx8) 73*4882a593Smuzhiyun.endm 74*4882a593Smuzhiyun 75*4882a593Smuzhiyunaddsub_return add add adc 76*4882a593Smuzhiyunaddsub_return sub sub sbb 77*4882a593Smuzhiyun 78*4882a593Smuzhiyun.macro incdec_return func ins insc 79*4882a593SmuzhiyunSYM_FUNC_START(atomic64_\func\()_return_cx8) 80*4882a593Smuzhiyun pushl %ebx 81*4882a593Smuzhiyun 82*4882a593Smuzhiyun read64 %esi 83*4882a593Smuzhiyun1: 84*4882a593Smuzhiyun movl %eax, %ebx 85*4882a593Smuzhiyun movl %edx, %ecx 86*4882a593Smuzhiyun \ins\()l $1, %ebx 87*4882a593Smuzhiyun \insc\()l $0, %ecx 88*4882a593Smuzhiyun LOCK_PREFIX 89*4882a593Smuzhiyun cmpxchg8b (%esi) 90*4882a593Smuzhiyun jne 1b 91*4882a593Smuzhiyun 92*4882a593Smuzhiyun10: 93*4882a593Smuzhiyun movl %ebx, %eax 94*4882a593Smuzhiyun movl %ecx, %edx 95*4882a593Smuzhiyun popl %ebx 96*4882a593Smuzhiyun RET 97*4882a593SmuzhiyunSYM_FUNC_END(atomic64_\func\()_return_cx8) 98*4882a593Smuzhiyun.endm 99*4882a593Smuzhiyun 100*4882a593Smuzhiyunincdec_return inc add adc 101*4882a593Smuzhiyunincdec_return dec sub sbb 102*4882a593Smuzhiyun 103*4882a593SmuzhiyunSYM_FUNC_START(atomic64_dec_if_positive_cx8) 104*4882a593Smuzhiyun pushl %ebx 105*4882a593Smuzhiyun 106*4882a593Smuzhiyun read64 %esi 107*4882a593Smuzhiyun1: 108*4882a593Smuzhiyun movl %eax, %ebx 109*4882a593Smuzhiyun movl %edx, %ecx 110*4882a593Smuzhiyun subl $1, %ebx 111*4882a593Smuzhiyun sbb $0, %ecx 112*4882a593Smuzhiyun js 2f 113*4882a593Smuzhiyun LOCK_PREFIX 114*4882a593Smuzhiyun cmpxchg8b (%esi) 115*4882a593Smuzhiyun jne 1b 116*4882a593Smuzhiyun 117*4882a593Smuzhiyun2: 118*4882a593Smuzhiyun movl %ebx, %eax 119*4882a593Smuzhiyun movl %ecx, %edx 120*4882a593Smuzhiyun popl %ebx 121*4882a593Smuzhiyun RET 122*4882a593SmuzhiyunSYM_FUNC_END(atomic64_dec_if_positive_cx8) 123*4882a593Smuzhiyun 124*4882a593SmuzhiyunSYM_FUNC_START(atomic64_add_unless_cx8) 125*4882a593Smuzhiyun pushl %ebp 126*4882a593Smuzhiyun pushl %ebx 127*4882a593Smuzhiyun/* these just push these two parameters on the stack */ 128*4882a593Smuzhiyun pushl %edi 129*4882a593Smuzhiyun pushl %ecx 130*4882a593Smuzhiyun 131*4882a593Smuzhiyun movl %eax, %ebp 132*4882a593Smuzhiyun movl %edx, %edi 133*4882a593Smuzhiyun 134*4882a593Smuzhiyun read64 %esi 135*4882a593Smuzhiyun1: 136*4882a593Smuzhiyun cmpl %eax, 0(%esp) 137*4882a593Smuzhiyun je 4f 138*4882a593Smuzhiyun2: 139*4882a593Smuzhiyun movl %eax, %ebx 140*4882a593Smuzhiyun movl %edx, %ecx 141*4882a593Smuzhiyun addl %ebp, %ebx 142*4882a593Smuzhiyun adcl %edi, %ecx 143*4882a593Smuzhiyun LOCK_PREFIX 144*4882a593Smuzhiyun cmpxchg8b (%esi) 145*4882a593Smuzhiyun jne 1b 146*4882a593Smuzhiyun 147*4882a593Smuzhiyun movl $1, %eax 148*4882a593Smuzhiyun3: 149*4882a593Smuzhiyun addl $8, %esp 150*4882a593Smuzhiyun popl %ebx 151*4882a593Smuzhiyun popl %ebp 152*4882a593Smuzhiyun RET 153*4882a593Smuzhiyun4: 154*4882a593Smuzhiyun cmpl %edx, 4(%esp) 155*4882a593Smuzhiyun jne 2b 156*4882a593Smuzhiyun xorl %eax, %eax 157*4882a593Smuzhiyun jmp 3b 158*4882a593SmuzhiyunSYM_FUNC_END(atomic64_add_unless_cx8) 159*4882a593Smuzhiyun 160*4882a593SmuzhiyunSYM_FUNC_START(atomic64_inc_not_zero_cx8) 161*4882a593Smuzhiyun pushl %ebx 162*4882a593Smuzhiyun 163*4882a593Smuzhiyun read64 %esi 164*4882a593Smuzhiyun1: 165*4882a593Smuzhiyun movl %eax, %ecx 166*4882a593Smuzhiyun orl %edx, %ecx 167*4882a593Smuzhiyun jz 3f 168*4882a593Smuzhiyun movl %eax, %ebx 169*4882a593Smuzhiyun xorl %ecx, %ecx 170*4882a593Smuzhiyun addl $1, %ebx 171*4882a593Smuzhiyun adcl %edx, %ecx 172*4882a593Smuzhiyun LOCK_PREFIX 173*4882a593Smuzhiyun cmpxchg8b (%esi) 174*4882a593Smuzhiyun jne 1b 175*4882a593Smuzhiyun 176*4882a593Smuzhiyun movl $1, %eax 177*4882a593Smuzhiyun3: 178*4882a593Smuzhiyun popl %ebx 179*4882a593Smuzhiyun RET 180*4882a593SmuzhiyunSYM_FUNC_END(atomic64_inc_not_zero_cx8) 181