1*4882a593Smuzhiyun #include <linux/string.h>
2*4882a593Smuzhiyun #include <linux/module.h>
3*4882a593Smuzhiyun #include <linux/io.h>
4*4882a593Smuzhiyun
5*4882a593Smuzhiyun #define movs(type,to,from) \
6*4882a593Smuzhiyun asm volatile("movs" type:"=&D" (to), "=&S" (from):"0" (to), "1" (from):"memory")
7*4882a593Smuzhiyun
8*4882a593Smuzhiyun /* Originally from i386/string.h */
rep_movs(void * to,const void * from,size_t n)9*4882a593Smuzhiyun static __always_inline void rep_movs(void *to, const void *from, size_t n)
10*4882a593Smuzhiyun {
11*4882a593Smuzhiyun unsigned long d0, d1, d2;
12*4882a593Smuzhiyun asm volatile("rep ; movsl\n\t"
13*4882a593Smuzhiyun "testb $2,%b4\n\t"
14*4882a593Smuzhiyun "je 1f\n\t"
15*4882a593Smuzhiyun "movsw\n"
16*4882a593Smuzhiyun "1:\ttestb $1,%b4\n\t"
17*4882a593Smuzhiyun "je 2f\n\t"
18*4882a593Smuzhiyun "movsb\n"
19*4882a593Smuzhiyun "2:"
20*4882a593Smuzhiyun : "=&c" (d0), "=&D" (d1), "=&S" (d2)
21*4882a593Smuzhiyun : "0" (n / 4), "q" (n), "1" ((long)to), "2" ((long)from)
22*4882a593Smuzhiyun : "memory");
23*4882a593Smuzhiyun }
24*4882a593Smuzhiyun
memcpy_fromio(void * to,const volatile void __iomem * from,size_t n)25*4882a593Smuzhiyun void memcpy_fromio(void *to, const volatile void __iomem *from, size_t n)
26*4882a593Smuzhiyun {
27*4882a593Smuzhiyun if (unlikely(!n))
28*4882a593Smuzhiyun return;
29*4882a593Smuzhiyun
30*4882a593Smuzhiyun /* Align any unaligned source IO */
31*4882a593Smuzhiyun if (unlikely(1 & (unsigned long)from)) {
32*4882a593Smuzhiyun movs("b", to, from);
33*4882a593Smuzhiyun n--;
34*4882a593Smuzhiyun }
35*4882a593Smuzhiyun if (n > 1 && unlikely(2 & (unsigned long)from)) {
36*4882a593Smuzhiyun movs("w", to, from);
37*4882a593Smuzhiyun n-=2;
38*4882a593Smuzhiyun }
39*4882a593Smuzhiyun rep_movs(to, (const void *)from, n);
40*4882a593Smuzhiyun }
41*4882a593Smuzhiyun EXPORT_SYMBOL(memcpy_fromio);
42*4882a593Smuzhiyun
memcpy_toio(volatile void __iomem * to,const void * from,size_t n)43*4882a593Smuzhiyun void memcpy_toio(volatile void __iomem *to, const void *from, size_t n)
44*4882a593Smuzhiyun {
45*4882a593Smuzhiyun if (unlikely(!n))
46*4882a593Smuzhiyun return;
47*4882a593Smuzhiyun
48*4882a593Smuzhiyun /* Align any unaligned destination IO */
49*4882a593Smuzhiyun if (unlikely(1 & (unsigned long)to)) {
50*4882a593Smuzhiyun movs("b", to, from);
51*4882a593Smuzhiyun n--;
52*4882a593Smuzhiyun }
53*4882a593Smuzhiyun if (n > 1 && unlikely(2 & (unsigned long)to)) {
54*4882a593Smuzhiyun movs("w", to, from);
55*4882a593Smuzhiyun n-=2;
56*4882a593Smuzhiyun }
57*4882a593Smuzhiyun rep_movs((void *)to, (const void *) from, n);
58*4882a593Smuzhiyun }
59*4882a593Smuzhiyun EXPORT_SYMBOL(memcpy_toio);
60*4882a593Smuzhiyun
memset_io(volatile void __iomem * a,int b,size_t c)61*4882a593Smuzhiyun void memset_io(volatile void __iomem *a, int b, size_t c)
62*4882a593Smuzhiyun {
63*4882a593Smuzhiyun /*
64*4882a593Smuzhiyun * TODO: memset can mangle the IO patterns quite a bit.
65*4882a593Smuzhiyun * perhaps it would be better to use a dumb one:
66*4882a593Smuzhiyun */
67*4882a593Smuzhiyun memset((void *)a, b, c);
68*4882a593Smuzhiyun }
69*4882a593Smuzhiyun EXPORT_SYMBOL(memset_io);
70