1*4882a593Smuzhiyun /* SPDX-License-Identifier: GPL-2.0 */
2*4882a593Smuzhiyun #ifndef __ASM_ARM_WORD_AT_A_TIME_H
3*4882a593Smuzhiyun #define __ASM_ARM_WORD_AT_A_TIME_H
4*4882a593Smuzhiyun
5*4882a593Smuzhiyun #ifndef __ARMEB__
6*4882a593Smuzhiyun
7*4882a593Smuzhiyun /*
8*4882a593Smuzhiyun * Little-endian word-at-a-time zero byte handling.
9*4882a593Smuzhiyun * Heavily based on the x86 algorithm.
10*4882a593Smuzhiyun */
11*4882a593Smuzhiyun #include <linux/kernel.h>
12*4882a593Smuzhiyun
13*4882a593Smuzhiyun struct word_at_a_time {
14*4882a593Smuzhiyun const unsigned long one_bits, high_bits;
15*4882a593Smuzhiyun };
16*4882a593Smuzhiyun
17*4882a593Smuzhiyun #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x01), REPEAT_BYTE(0x80) }
18*4882a593Smuzhiyun
has_zero(unsigned long a,unsigned long * bits,const struct word_at_a_time * c)19*4882a593Smuzhiyun static inline unsigned long has_zero(unsigned long a, unsigned long *bits,
20*4882a593Smuzhiyun const struct word_at_a_time *c)
21*4882a593Smuzhiyun {
22*4882a593Smuzhiyun unsigned long mask = ((a - c->one_bits) & ~a) & c->high_bits;
23*4882a593Smuzhiyun *bits = mask;
24*4882a593Smuzhiyun return mask;
25*4882a593Smuzhiyun }
26*4882a593Smuzhiyun
27*4882a593Smuzhiyun #define prep_zero_mask(a, bits, c) (bits)
28*4882a593Smuzhiyun
create_zero_mask(unsigned long bits)29*4882a593Smuzhiyun static inline unsigned long create_zero_mask(unsigned long bits)
30*4882a593Smuzhiyun {
31*4882a593Smuzhiyun bits = (bits - 1) & ~bits;
32*4882a593Smuzhiyun return bits >> 7;
33*4882a593Smuzhiyun }
34*4882a593Smuzhiyun
find_zero(unsigned long mask)35*4882a593Smuzhiyun static inline unsigned long find_zero(unsigned long mask)
36*4882a593Smuzhiyun {
37*4882a593Smuzhiyun unsigned long ret;
38*4882a593Smuzhiyun
39*4882a593Smuzhiyun #if __LINUX_ARM_ARCH__ >= 5
40*4882a593Smuzhiyun /* We have clz available. */
41*4882a593Smuzhiyun ret = fls(mask) >> 3;
42*4882a593Smuzhiyun #else
43*4882a593Smuzhiyun /* (000000 0000ff 00ffff ffffff) -> ( 1 1 2 3 ) */
44*4882a593Smuzhiyun ret = (0x0ff0001 + mask) >> 23;
45*4882a593Smuzhiyun /* Fix the 1 for 00 case */
46*4882a593Smuzhiyun ret &= mask;
47*4882a593Smuzhiyun #endif
48*4882a593Smuzhiyun
49*4882a593Smuzhiyun return ret;
50*4882a593Smuzhiyun }
51*4882a593Smuzhiyun
52*4882a593Smuzhiyun #define zero_bytemask(mask) (mask)
53*4882a593Smuzhiyun
54*4882a593Smuzhiyun #else /* __ARMEB__ */
55*4882a593Smuzhiyun #include <asm-generic/word-at-a-time.h>
56*4882a593Smuzhiyun #endif
57*4882a593Smuzhiyun
58*4882a593Smuzhiyun #ifdef CONFIG_DCACHE_WORD_ACCESS
59*4882a593Smuzhiyun
60*4882a593Smuzhiyun /*
61*4882a593Smuzhiyun * Load an unaligned word from kernel space.
62*4882a593Smuzhiyun *
63*4882a593Smuzhiyun * In the (very unlikely) case of the word being a page-crosser
64*4882a593Smuzhiyun * and the next page not being mapped, take the exception and
65*4882a593Smuzhiyun * return zeroes in the non-existing part.
66*4882a593Smuzhiyun */
load_unaligned_zeropad(const void * addr)67*4882a593Smuzhiyun static inline unsigned long load_unaligned_zeropad(const void *addr)
68*4882a593Smuzhiyun {
69*4882a593Smuzhiyun unsigned long ret, offset;
70*4882a593Smuzhiyun
71*4882a593Smuzhiyun /* Load word from unaligned pointer addr */
72*4882a593Smuzhiyun asm(
73*4882a593Smuzhiyun "1: ldr %0, [%2]\n"
74*4882a593Smuzhiyun "2:\n"
75*4882a593Smuzhiyun " .pushsection .text.fixup,\"ax\"\n"
76*4882a593Smuzhiyun " .align 2\n"
77*4882a593Smuzhiyun "3: and %1, %2, #0x3\n"
78*4882a593Smuzhiyun " bic %2, %2, #0x3\n"
79*4882a593Smuzhiyun " ldr %0, [%2]\n"
80*4882a593Smuzhiyun " lsl %1, %1, #0x3\n"
81*4882a593Smuzhiyun #ifndef __ARMEB__
82*4882a593Smuzhiyun " lsr %0, %0, %1\n"
83*4882a593Smuzhiyun #else
84*4882a593Smuzhiyun " lsl %0, %0, %1\n"
85*4882a593Smuzhiyun #endif
86*4882a593Smuzhiyun " b 2b\n"
87*4882a593Smuzhiyun " .popsection\n"
88*4882a593Smuzhiyun " .pushsection __ex_table,\"a\"\n"
89*4882a593Smuzhiyun " .align 3\n"
90*4882a593Smuzhiyun " .long 1b, 3b\n"
91*4882a593Smuzhiyun " .popsection"
92*4882a593Smuzhiyun : "=&r" (ret), "=&r" (offset)
93*4882a593Smuzhiyun : "r" (addr), "Qo" (*(unsigned long *)addr));
94*4882a593Smuzhiyun
95*4882a593Smuzhiyun return ret;
96*4882a593Smuzhiyun }
97*4882a593Smuzhiyun
98*4882a593Smuzhiyun #endif /* DCACHE_WORD_ACCESS */
99*4882a593Smuzhiyun #endif /* __ASM_ARM_WORD_AT_A_TIME_H */
100