1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0 */
2*4882a593Smuzhiyun #ifndef _ASM_WORD_AT_A_TIME_H
3*4882a593Smuzhiyun #define _ASM_WORD_AT_A_TIME_H
4*4882a593Smuzhiyun
5*4882a593Smuzhiyun #include <linux/kernel.h>
6*4882a593Smuzhiyun #include <asm/byteorder.h>
7*4882a593Smuzhiyun
8*4882a593Smuzhiyun #ifdef __BIG_ENDIAN
9*4882a593Smuzhiyun
10*4882a593Smuzhiyun struct word_at_a_time {
11*4882a593Smuzhiyun const unsigned long high_bits, low_bits;
12*4882a593Smuzhiyun };
13*4882a593Smuzhiyun
14*4882a593Smuzhiyun #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0xfe) + 1, REPEAT_BYTE(0x7f) }
15*4882a593Smuzhiyun
16*4882a593Smuzhiyun /* Bit set in the bytes that have a zero */
prep_zero_mask(unsigned long val,unsigned long rhs,const struct word_at_a_time * c)17*4882a593Smuzhiyun static inline long prep_zero_mask(unsigned long val, unsigned long rhs, const struct word_at_a_time *c)
18*4882a593Smuzhiyun {
19*4882a593Smuzhiyun unsigned long mask = (val & c->low_bits) + c->low_bits;
20*4882a593Smuzhiyun return ~(mask | rhs);
21*4882a593Smuzhiyun }
22*4882a593Smuzhiyun
23*4882a593Smuzhiyun #define create_zero_mask(mask) (mask)
24*4882a593Smuzhiyun
find_zero(unsigned long mask)25*4882a593Smuzhiyun static inline long find_zero(unsigned long mask)
26*4882a593Smuzhiyun {
27*4882a593Smuzhiyun long byte = 0;
28*4882a593Smuzhiyun #ifdef CONFIG_64BIT
29*4882a593Smuzhiyun if (mask >> 32)
30*4882a593Smuzhiyun mask >>= 32;
31*4882a593Smuzhiyun else
32*4882a593Smuzhiyun byte = 4;
33*4882a593Smuzhiyun #endif
34*4882a593Smuzhiyun if (mask >> 16)
35*4882a593Smuzhiyun mask >>= 16;
36*4882a593Smuzhiyun else
37*4882a593Smuzhiyun byte += 2;
38*4882a593Smuzhiyun return (mask >> 8) ? byte : byte + 1;
39*4882a593Smuzhiyun }
40*4882a593Smuzhiyun
has_zero(unsigned long val,unsigned long * data,const struct word_at_a_time * c)41*4882a593Smuzhiyun static inline bool has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c)
42*4882a593Smuzhiyun {
43*4882a593Smuzhiyun unsigned long rhs = val | c->low_bits;
44*4882a593Smuzhiyun *data = rhs;
45*4882a593Smuzhiyun return (val + c->high_bits) & ~rhs;
46*4882a593Smuzhiyun }
47*4882a593Smuzhiyun
48*4882a593Smuzhiyun #ifndef zero_bytemask
49*4882a593Smuzhiyun #define zero_bytemask(mask) (~1ul << __fls(mask))
50*4882a593Smuzhiyun #endif
51*4882a593Smuzhiyun
52*4882a593Smuzhiyun #else
53*4882a593Smuzhiyun
54*4882a593Smuzhiyun /*
55*4882a593Smuzhiyun * The optimal byte mask counting is probably going to be something
56*4882a593Smuzhiyun * that is architecture-specific. If you have a reliably fast
57*4882a593Smuzhiyun * bit count instruction, that might be better than the multiply
58*4882a593Smuzhiyun * and shift, for example.
59*4882a593Smuzhiyun */
60*4882a593Smuzhiyun struct word_at_a_time {
61*4882a593Smuzhiyun const unsigned long one_bits, high_bits;
62*4882a593Smuzhiyun };
63*4882a593Smuzhiyun
64*4882a593Smuzhiyun #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
65*4882a593Smuzhiyun
66*4882a593Smuzhiyun #ifdef CONFIG_64BIT
67*4882a593Smuzhiyun
68*4882a593Smuzhiyun /*
69*4882a593Smuzhiyun * Jan Achrenius on G+: microoptimized version of
70*4882a593Smuzhiyun * the simpler "(mask & ONEBYTES) * ONEBYTES >> 56"
71*4882a593Smuzhiyun * that works for the bytemasks without having to
72*4882a593Smuzhiyun * mask them first.
73*4882a593Smuzhiyun */
count_masked_bytes(unsigned long mask)74*4882a593Smuzhiyun static inline long count_masked_bytes(unsigned long mask)
75*4882a593Smuzhiyun {
76*4882a593Smuzhiyun return mask*0x0001020304050608ul >> 56;
77*4882a593Smuzhiyun }
78*4882a593Smuzhiyun
79*4882a593Smuzhiyun #else /* 32-bit case */
80*4882a593Smuzhiyun
81*4882a593Smuzhiyun /* Carl Chatfield / Jan Achrenius G+ version for 32-bit */
count_masked_bytes(long mask)82*4882a593Smuzhiyun static inline long count_masked_bytes(long mask)
83*4882a593Smuzhiyun {
84*4882a593Smuzhiyun /* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */
85*4882a593Smuzhiyun long a = (0x0ff0001+mask) >> 23;
86*4882a593Smuzhiyun /* Fix the 1 for 00 case */
87*4882a593Smuzhiyun return a & mask;
88*4882a593Smuzhiyun }
89*4882a593Smuzhiyun
90*4882a593Smuzhiyun #endif
91*4882a593Smuzhiyun
92*4882a593Smuzhiyun /* Return nonzero if it has a zero */
has_zero(unsigned long a,unsigned long * bits,const struct word_at_a_time * c)93*4882a593Smuzhiyun static inline unsigned long has_zero(unsigned long a, unsigned long *bits, const struct word_at_a_time *c)
94*4882a593Smuzhiyun {
95*4882a593Smuzhiyun unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
96*4882a593Smuzhiyun *bits = mask;
97*4882a593Smuzhiyun return mask;
98*4882a593Smuzhiyun }
99*4882a593Smuzhiyun
prep_zero_mask(unsigned long a,unsigned long bits,const struct word_at_a_time * c)100*4882a593Smuzhiyun static inline unsigned long prep_zero_mask(unsigned long a, unsigned long bits, const struct word_at_a_time *c)
101*4882a593Smuzhiyun {
102*4882a593Smuzhiyun return bits;
103*4882a593Smuzhiyun }
104*4882a593Smuzhiyun
create_zero_mask(unsigned long bits)105*4882a593Smuzhiyun static inline unsigned long create_zero_mask(unsigned long bits)
106*4882a593Smuzhiyun {
107*4882a593Smuzhiyun bits = (bits - 1) & ~bits;
108*4882a593Smuzhiyun return bits >> 7;
109*4882a593Smuzhiyun }
110*4882a593Smuzhiyun
111*4882a593Smuzhiyun /* The mask we created is directly usable as a bytemask */
112*4882a593Smuzhiyun #define zero_bytemask(mask) (mask)
113*4882a593Smuzhiyun
find_zero(unsigned long mask)114*4882a593Smuzhiyun static inline unsigned long find_zero(unsigned long mask)
115*4882a593Smuzhiyun {
116*4882a593Smuzhiyun return count_masked_bytes(mask);
117*4882a593Smuzhiyun }
118*4882a593Smuzhiyun
119*4882a593Smuzhiyun #endif /* __BIG_ENDIAN */
120*4882a593Smuzhiyun
121*4882a593Smuzhiyun #endif /* _ASM_WORD_AT_A_TIME_H */
122