1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0 */ 2*4882a593Smuzhiyun #ifndef _SPARC64_BACKOFF_H 3*4882a593Smuzhiyun #define _SPARC64_BACKOFF_H 4*4882a593Smuzhiyun 5*4882a593Smuzhiyun /* The macros in this file implement an exponential backoff facility 6*4882a593Smuzhiyun * for atomic operations. 7*4882a593Smuzhiyun * 8*4882a593Smuzhiyun * When multiple threads compete on an atomic operation, it is 9*4882a593Smuzhiyun * possible for one thread to be continually denied a successful 10*4882a593Smuzhiyun * completion of the compare-and-swap instruction. Heavily 11*4882a593Smuzhiyun * threaded cpu implementations like Niagara can compound this 12*4882a593Smuzhiyun * problem even further. 13*4882a593Smuzhiyun * 14*4882a593Smuzhiyun * When an atomic operation fails and needs to be retried, we spin a 15*4882a593Smuzhiyun * certain number of times. At each subsequent failure of the same 16*4882a593Smuzhiyun * operation we double the spin count, realizing an exponential 17*4882a593Smuzhiyun * backoff. 18*4882a593Smuzhiyun * 19*4882a593Smuzhiyun * When we spin, we try to use an operation that will cause the 20*4882a593Smuzhiyun * current cpu strand to block, and therefore make the core fully 21*4882a593Smuzhiyun * available to any other other runnable strands. There are two 22*4882a593Smuzhiyun * options, based upon cpu capabilities. 23*4882a593Smuzhiyun * 24*4882a593Smuzhiyun * On all cpus prior to SPARC-T4 we do three dummy reads of the 25*4882a593Smuzhiyun * condition code register. Each read blocks the strand for something 26*4882a593Smuzhiyun * between 40 and 50 cpu cycles. 27*4882a593Smuzhiyun * 28*4882a593Smuzhiyun * For SPARC-T4 and later we have a special "pause" instruction 29*4882a593Smuzhiyun * available. This is implemented using writes to register %asr27. 30*4882a593Smuzhiyun * The cpu will block the number of cycles written into the register, 31*4882a593Smuzhiyun * unless a disrupting trap happens first. SPARC-T4 specifically 32*4882a593Smuzhiyun * implements pause with a granularity of 8 cycles. Each strand has 33*4882a593Smuzhiyun * an internal pause counter which decrements every 8 cycles. So the 34*4882a593Smuzhiyun * chip shifts the %asr27 value down by 3 bits, and writes the result 35*4882a593Smuzhiyun * into the pause counter. If a value smaller than 8 is written, the 36*4882a593Smuzhiyun * chip blocks for 1 cycle. 37*4882a593Smuzhiyun * 38*4882a593Smuzhiyun * To achieve the same amount of backoff as the three %ccr reads give 39*4882a593Smuzhiyun * on earlier chips, we shift the backoff value up by 7 bits. (Three 40*4882a593Smuzhiyun * %ccr reads block for about 128 cycles, 1 << 7 == 128) We write the 41*4882a593Smuzhiyun * whole amount we want to block into the pause register, rather than 42*4882a593Smuzhiyun * loop writing 128 each time. 43*4882a593Smuzhiyun */ 44*4882a593Smuzhiyun 45*4882a593Smuzhiyun #define BACKOFF_LIMIT (4 * 1024) 46*4882a593Smuzhiyun 47*4882a593Smuzhiyun #ifdef CONFIG_SMP 48*4882a593Smuzhiyun 49*4882a593Smuzhiyun #define BACKOFF_SETUP(reg) \ 50*4882a593Smuzhiyun mov 1, reg 51*4882a593Smuzhiyun 52*4882a593Smuzhiyun #define BACKOFF_LABEL(spin_label, continue_label) \ 53*4882a593Smuzhiyun spin_label 54*4882a593Smuzhiyun 55*4882a593Smuzhiyun #define BACKOFF_SPIN(reg, tmp, label) \ 56*4882a593Smuzhiyun mov reg, tmp; \ 57*4882a593Smuzhiyun 88: rd %ccr, %g0; \ 58*4882a593Smuzhiyun rd %ccr, %g0; \ 59*4882a593Smuzhiyun rd %ccr, %g0; \ 60*4882a593Smuzhiyun .section .pause_3insn_patch,"ax";\ 61*4882a593Smuzhiyun .word 88b; \ 62*4882a593Smuzhiyun sllx tmp, 7, tmp; \ 63*4882a593Smuzhiyun wr tmp, 0, %asr27; \ 64*4882a593Smuzhiyun clr tmp; \ 65*4882a593Smuzhiyun .previous; \ 66*4882a593Smuzhiyun brnz,pt tmp, 88b; \ 67*4882a593Smuzhiyun sub tmp, 1, tmp; \ 68*4882a593Smuzhiyun set BACKOFF_LIMIT, tmp; \ 69*4882a593Smuzhiyun cmp reg, tmp; \ 70*4882a593Smuzhiyun bg,pn %xcc, label; \ 71*4882a593Smuzhiyun nop; \ 72*4882a593Smuzhiyun ba,pt %xcc, label; \ 73*4882a593Smuzhiyun sllx reg, 1, reg; 74*4882a593Smuzhiyun 75*4882a593Smuzhiyun #else 76*4882a593Smuzhiyun 77*4882a593Smuzhiyun #define BACKOFF_SETUP(reg) 78*4882a593Smuzhiyun 79*4882a593Smuzhiyun #define BACKOFF_LABEL(spin_label, continue_label) \ 80*4882a593Smuzhiyun continue_label 81*4882a593Smuzhiyun 82*4882a593Smuzhiyun #define BACKOFF_SPIN(reg, tmp, label) 83*4882a593Smuzhiyun 84*4882a593Smuzhiyun #endif 85*4882a593Smuzhiyun 86*4882a593Smuzhiyun #endif /* _SPARC64_BACKOFF_H */ 87