1*4882a593Smuzhiyun/* SPDX-License-Identifier: GPL-2.0 */ 2*4882a593Smuzhiyun/* bzero.S: Simple prefetching memset, bzero, and clear_user 3*4882a593Smuzhiyun * implementations. 4*4882a593Smuzhiyun * 5*4882a593Smuzhiyun * Copyright (C) 2005 David S. Miller <davem@davemloft.net> 6*4882a593Smuzhiyun */ 7*4882a593Smuzhiyun 8*4882a593Smuzhiyun#include <linux/linkage.h> 9*4882a593Smuzhiyun#include <asm/export.h> 10*4882a593Smuzhiyun 11*4882a593Smuzhiyun .text 12*4882a593Smuzhiyun 13*4882a593SmuzhiyunENTRY(memset) /* %o0=buf, %o1=pat, %o2=len */ 14*4882a593Smuzhiyun and %o1, 0xff, %o3 15*4882a593Smuzhiyun mov %o2, %o1 16*4882a593Smuzhiyun sllx %o3, 8, %g1 17*4882a593Smuzhiyun or %g1, %o3, %o2 18*4882a593Smuzhiyun sllx %o2, 16, %g1 19*4882a593Smuzhiyun or %g1, %o2, %o2 20*4882a593Smuzhiyun sllx %o2, 32, %g1 21*4882a593Smuzhiyun ba,pt %xcc, 1f 22*4882a593Smuzhiyun or %g1, %o2, %o2 23*4882a593Smuzhiyun 24*4882a593SmuzhiyunENTRY(__bzero) /* %o0=buf, %o1=len */ 25*4882a593Smuzhiyun clr %o2 26*4882a593Smuzhiyun1: mov %o0, %o3 27*4882a593Smuzhiyun brz,pn %o1, __bzero_done 28*4882a593Smuzhiyun cmp %o1, 16 29*4882a593Smuzhiyun bl,pn %icc, __bzero_tiny 30*4882a593Smuzhiyun prefetch [%o0 + 0x000], #n_writes 31*4882a593Smuzhiyun andcc %o0, 0x3, %g0 32*4882a593Smuzhiyun be,pt %icc, 2f 33*4882a593Smuzhiyun1: stb %o2, [%o0 + 0x00] 34*4882a593Smuzhiyun add %o0, 1, %o0 35*4882a593Smuzhiyun andcc %o0, 0x3, %g0 36*4882a593Smuzhiyun bne,pn %icc, 1b 37*4882a593Smuzhiyun sub %o1, 1, %o1 38*4882a593Smuzhiyun2: andcc %o0, 0x7, %g0 39*4882a593Smuzhiyun be,pt %icc, 3f 40*4882a593Smuzhiyun stw %o2, [%o0 + 0x00] 41*4882a593Smuzhiyun sub %o1, 4, %o1 42*4882a593Smuzhiyun add %o0, 4, %o0 43*4882a593Smuzhiyun3: and %o1, 0x38, %g1 44*4882a593Smuzhiyun cmp %o1, 0x40 45*4882a593Smuzhiyun andn %o1, 0x3f, %o4 46*4882a593Smuzhiyun bl,pn %icc, 5f 47*4882a593Smuzhiyun and %o1, 0x7, %o1 48*4882a593Smuzhiyun prefetch [%o0 + 0x040], #n_writes 49*4882a593Smuzhiyun prefetch [%o0 + 0x080], #n_writes 50*4882a593Smuzhiyun prefetch [%o0 + 0x0c0], #n_writes 51*4882a593Smuzhiyun prefetch [%o0 + 0x100], #n_writes 52*4882a593Smuzhiyun prefetch [%o0 + 0x140], #n_writes 53*4882a593Smuzhiyun4: prefetch [%o0 + 0x180], #n_writes 54*4882a593Smuzhiyun stx %o2, [%o0 + 0x00] 55*4882a593Smuzhiyun stx %o2, [%o0 + 0x08] 56*4882a593Smuzhiyun stx %o2, [%o0 + 0x10] 57*4882a593Smuzhiyun stx %o2, [%o0 + 0x18] 58*4882a593Smuzhiyun stx %o2, [%o0 + 0x20] 59*4882a593Smuzhiyun stx %o2, [%o0 + 0x28] 60*4882a593Smuzhiyun stx %o2, [%o0 + 0x30] 61*4882a593Smuzhiyun stx %o2, [%o0 + 0x38] 62*4882a593Smuzhiyun subcc %o4, 0x40, %o4 63*4882a593Smuzhiyun bne,pt %icc, 4b 64*4882a593Smuzhiyun add %o0, 0x40, %o0 65*4882a593Smuzhiyun brz,pn %g1, 6f 66*4882a593Smuzhiyun nop 67*4882a593Smuzhiyun5: stx %o2, [%o0 + 0x00] 68*4882a593Smuzhiyun subcc %g1, 8, %g1 69*4882a593Smuzhiyun bne,pt %icc, 5b 70*4882a593Smuzhiyun add %o0, 0x8, %o0 71*4882a593Smuzhiyun6: brz,pt %o1, __bzero_done 72*4882a593Smuzhiyun nop 73*4882a593Smuzhiyun__bzero_tiny: 74*4882a593Smuzhiyun1: stb %o2, [%o0 + 0x00] 75*4882a593Smuzhiyun subcc %o1, 1, %o1 76*4882a593Smuzhiyun bne,pt %icc, 1b 77*4882a593Smuzhiyun add %o0, 1, %o0 78*4882a593Smuzhiyun__bzero_done: 79*4882a593Smuzhiyun retl 80*4882a593Smuzhiyun mov %o3, %o0 81*4882a593SmuzhiyunENDPROC(__bzero) 82*4882a593SmuzhiyunENDPROC(memset) 83*4882a593SmuzhiyunEXPORT_SYMBOL(__bzero) 84*4882a593SmuzhiyunEXPORT_SYMBOL(memset) 85*4882a593Smuzhiyun 86*4882a593Smuzhiyun#define EX_ST(x,y) \ 87*4882a593Smuzhiyun98: x,y; \ 88*4882a593Smuzhiyun .section __ex_table,"a";\ 89*4882a593Smuzhiyun .align 4; \ 90*4882a593Smuzhiyun .word 98b, __retl_o1; \ 91*4882a593Smuzhiyun .text; \ 92*4882a593Smuzhiyun .align 4; 93*4882a593Smuzhiyun 94*4882a593SmuzhiyunENTRY(__clear_user) /* %o0=buf, %o1=len */ 95*4882a593Smuzhiyun brz,pn %o1, __clear_user_done 96*4882a593Smuzhiyun cmp %o1, 16 97*4882a593Smuzhiyun bl,pn %icc, __clear_user_tiny 98*4882a593Smuzhiyun EX_ST(prefetcha [%o0 + 0x00] %asi, #n_writes) 99*4882a593Smuzhiyun andcc %o0, 0x3, %g0 100*4882a593Smuzhiyun be,pt %icc, 2f 101*4882a593Smuzhiyun1: EX_ST(stba %g0, [%o0 + 0x00] %asi) 102*4882a593Smuzhiyun add %o0, 1, %o0 103*4882a593Smuzhiyun andcc %o0, 0x3, %g0 104*4882a593Smuzhiyun bne,pn %icc, 1b 105*4882a593Smuzhiyun sub %o1, 1, %o1 106*4882a593Smuzhiyun2: andcc %o0, 0x7, %g0 107*4882a593Smuzhiyun be,pt %icc, 3f 108*4882a593Smuzhiyun EX_ST(stwa %g0, [%o0 + 0x00] %asi) 109*4882a593Smuzhiyun sub %o1, 4, %o1 110*4882a593Smuzhiyun add %o0, 4, %o0 111*4882a593Smuzhiyun3: and %o1, 0x38, %g1 112*4882a593Smuzhiyun cmp %o1, 0x40 113*4882a593Smuzhiyun andn %o1, 0x3f, %o4 114*4882a593Smuzhiyun bl,pn %icc, 5f 115*4882a593Smuzhiyun and %o1, 0x7, %o1 116*4882a593Smuzhiyun EX_ST(prefetcha [%o0 + 0x040] %asi, #n_writes) 117*4882a593Smuzhiyun EX_ST(prefetcha [%o0 + 0x080] %asi, #n_writes) 118*4882a593Smuzhiyun EX_ST(prefetcha [%o0 + 0x0c0] %asi, #n_writes) 119*4882a593Smuzhiyun EX_ST(prefetcha [%o0 + 0x100] %asi, #n_writes) 120*4882a593Smuzhiyun EX_ST(prefetcha [%o0 + 0x140] %asi, #n_writes) 121*4882a593Smuzhiyun4: EX_ST(prefetcha [%o0 + 0x180] %asi, #n_writes) 122*4882a593Smuzhiyun EX_ST(stxa %g0, [%o0 + 0x00] %asi) 123*4882a593Smuzhiyun EX_ST(stxa %g0, [%o0 + 0x08] %asi) 124*4882a593Smuzhiyun EX_ST(stxa %g0, [%o0 + 0x10] %asi) 125*4882a593Smuzhiyun EX_ST(stxa %g0, [%o0 + 0x18] %asi) 126*4882a593Smuzhiyun EX_ST(stxa %g0, [%o0 + 0x20] %asi) 127*4882a593Smuzhiyun EX_ST(stxa %g0, [%o0 + 0x28] %asi) 128*4882a593Smuzhiyun EX_ST(stxa %g0, [%o0 + 0x30] %asi) 129*4882a593Smuzhiyun EX_ST(stxa %g0, [%o0 + 0x38] %asi) 130*4882a593Smuzhiyun subcc %o4, 0x40, %o4 131*4882a593Smuzhiyun bne,pt %icc, 4b 132*4882a593Smuzhiyun add %o0, 0x40, %o0 133*4882a593Smuzhiyun brz,pn %g1, 6f 134*4882a593Smuzhiyun nop 135*4882a593Smuzhiyun5: EX_ST(stxa %g0, [%o0 + 0x00] %asi) 136*4882a593Smuzhiyun subcc %g1, 8, %g1 137*4882a593Smuzhiyun bne,pt %icc, 5b 138*4882a593Smuzhiyun add %o0, 0x8, %o0 139*4882a593Smuzhiyun6: brz,pt %o1, __clear_user_done 140*4882a593Smuzhiyun nop 141*4882a593Smuzhiyun__clear_user_tiny: 142*4882a593Smuzhiyun1: EX_ST(stba %g0, [%o0 + 0x00] %asi) 143*4882a593Smuzhiyun subcc %o1, 1, %o1 144*4882a593Smuzhiyun bne,pt %icc, 1b 145*4882a593Smuzhiyun add %o0, 1, %o0 146*4882a593Smuzhiyun__clear_user_done: 147*4882a593Smuzhiyun retl 148*4882a593Smuzhiyun clr %o0 149*4882a593SmuzhiyunENDPROC(__clear_user) 150*4882a593SmuzhiyunEXPORT_SYMBOL(__clear_user) 151