1*4882a593Smuzhiyun/* SPDX-License-Identifier: GPL-2.0 */ 2*4882a593Smuzhiyun/* 3*4882a593Smuzhiyun * String handling functions. 4*4882a593Smuzhiyun * 5*4882a593Smuzhiyun * Copyright IBM Corp. 2012 6*4882a593Smuzhiyun */ 7*4882a593Smuzhiyun 8*4882a593Smuzhiyun#include <linux/linkage.h> 9*4882a593Smuzhiyun#include <asm/export.h> 10*4882a593Smuzhiyun#include <asm/nospec-insn.h> 11*4882a593Smuzhiyun 12*4882a593Smuzhiyun GEN_BR_THUNK %r14 13*4882a593Smuzhiyun 14*4882a593Smuzhiyun/* 15*4882a593Smuzhiyun * void *memmove(void *dest, const void *src, size_t n) 16*4882a593Smuzhiyun */ 17*4882a593SmuzhiyunWEAK(memmove) 18*4882a593SmuzhiyunENTRY(__memmove) 19*4882a593Smuzhiyun ltgr %r4,%r4 20*4882a593Smuzhiyun lgr %r1,%r2 21*4882a593Smuzhiyun jz .Lmemmove_exit 22*4882a593Smuzhiyun aghi %r4,-1 23*4882a593Smuzhiyun clgr %r2,%r3 24*4882a593Smuzhiyun jnh .Lmemmove_forward 25*4882a593Smuzhiyun la %r5,1(%r4,%r3) 26*4882a593Smuzhiyun clgr %r2,%r5 27*4882a593Smuzhiyun jl .Lmemmove_reverse 28*4882a593Smuzhiyun.Lmemmove_forward: 29*4882a593Smuzhiyun srlg %r0,%r4,8 30*4882a593Smuzhiyun ltgr %r0,%r0 31*4882a593Smuzhiyun jz .Lmemmove_forward_remainder 32*4882a593Smuzhiyun.Lmemmove_forward_loop: 33*4882a593Smuzhiyun mvc 0(256,%r1),0(%r3) 34*4882a593Smuzhiyun la %r1,256(%r1) 35*4882a593Smuzhiyun la %r3,256(%r3) 36*4882a593Smuzhiyun brctg %r0,.Lmemmove_forward_loop 37*4882a593Smuzhiyun.Lmemmove_forward_remainder: 38*4882a593Smuzhiyun larl %r5,.Lmemmove_mvc 39*4882a593Smuzhiyun ex %r4,0(%r5) 40*4882a593Smuzhiyun.Lmemmove_exit: 41*4882a593Smuzhiyun BR_EX %r14 42*4882a593Smuzhiyun.Lmemmove_reverse: 43*4882a593Smuzhiyun ic %r0,0(%r4,%r3) 44*4882a593Smuzhiyun stc %r0,0(%r4,%r1) 45*4882a593Smuzhiyun brctg %r4,.Lmemmove_reverse 46*4882a593Smuzhiyun ic %r0,0(%r4,%r3) 47*4882a593Smuzhiyun stc %r0,0(%r4,%r1) 48*4882a593Smuzhiyun BR_EX %r14 49*4882a593Smuzhiyun.Lmemmove_mvc: 50*4882a593Smuzhiyun mvc 0(1,%r1),0(%r3) 51*4882a593SmuzhiyunENDPROC(__memmove) 52*4882a593SmuzhiyunEXPORT_SYMBOL(memmove) 53*4882a593Smuzhiyun 54*4882a593Smuzhiyun/* 55*4882a593Smuzhiyun * memset implementation 56*4882a593Smuzhiyun * 57*4882a593Smuzhiyun * This code corresponds to the C construct below. We do distinguish 58*4882a593Smuzhiyun * between clearing (c == 0) and setting a memory array (c != 0) simply 59*4882a593Smuzhiyun * because nearly all memset invocations in the kernel clear memory and 60*4882a593Smuzhiyun * the xc instruction is preferred in such cases. 61*4882a593Smuzhiyun * 62*4882a593Smuzhiyun * void *memset(void *s, int c, size_t n) 63*4882a593Smuzhiyun * { 64*4882a593Smuzhiyun * if (likely(c == 0)) 65*4882a593Smuzhiyun * return __builtin_memset(s, 0, n); 66*4882a593Smuzhiyun * return __builtin_memset(s, c, n); 67*4882a593Smuzhiyun * } 68*4882a593Smuzhiyun */ 69*4882a593SmuzhiyunWEAK(memset) 70*4882a593SmuzhiyunENTRY(__memset) 71*4882a593Smuzhiyun ltgr %r4,%r4 72*4882a593Smuzhiyun jz .Lmemset_exit 73*4882a593Smuzhiyun ltgr %r3,%r3 74*4882a593Smuzhiyun jnz .Lmemset_fill 75*4882a593Smuzhiyun aghi %r4,-1 76*4882a593Smuzhiyun srlg %r3,%r4,8 77*4882a593Smuzhiyun ltgr %r3,%r3 78*4882a593Smuzhiyun lgr %r1,%r2 79*4882a593Smuzhiyun jz .Lmemset_clear_remainder 80*4882a593Smuzhiyun.Lmemset_clear_loop: 81*4882a593Smuzhiyun xc 0(256,%r1),0(%r1) 82*4882a593Smuzhiyun la %r1,256(%r1) 83*4882a593Smuzhiyun brctg %r3,.Lmemset_clear_loop 84*4882a593Smuzhiyun.Lmemset_clear_remainder: 85*4882a593Smuzhiyun larl %r3,.Lmemset_xc 86*4882a593Smuzhiyun ex %r4,0(%r3) 87*4882a593Smuzhiyun.Lmemset_exit: 88*4882a593Smuzhiyun BR_EX %r14 89*4882a593Smuzhiyun.Lmemset_fill: 90*4882a593Smuzhiyun cghi %r4,1 91*4882a593Smuzhiyun lgr %r1,%r2 92*4882a593Smuzhiyun je .Lmemset_fill_exit 93*4882a593Smuzhiyun aghi %r4,-2 94*4882a593Smuzhiyun srlg %r5,%r4,8 95*4882a593Smuzhiyun ltgr %r5,%r5 96*4882a593Smuzhiyun jz .Lmemset_fill_remainder 97*4882a593Smuzhiyun.Lmemset_fill_loop: 98*4882a593Smuzhiyun stc %r3,0(%r1) 99*4882a593Smuzhiyun mvc 1(255,%r1),0(%r1) 100*4882a593Smuzhiyun la %r1,256(%r1) 101*4882a593Smuzhiyun brctg %r5,.Lmemset_fill_loop 102*4882a593Smuzhiyun.Lmemset_fill_remainder: 103*4882a593Smuzhiyun stc %r3,0(%r1) 104*4882a593Smuzhiyun larl %r5,.Lmemset_mvc 105*4882a593Smuzhiyun ex %r4,0(%r5) 106*4882a593Smuzhiyun BR_EX %r14 107*4882a593Smuzhiyun.Lmemset_fill_exit: 108*4882a593Smuzhiyun stc %r3,0(%r1) 109*4882a593Smuzhiyun BR_EX %r14 110*4882a593Smuzhiyun.Lmemset_xc: 111*4882a593Smuzhiyun xc 0(1,%r1),0(%r1) 112*4882a593Smuzhiyun.Lmemset_mvc: 113*4882a593Smuzhiyun mvc 1(1,%r1),0(%r1) 114*4882a593SmuzhiyunENDPROC(__memset) 115*4882a593SmuzhiyunEXPORT_SYMBOL(memset) 116*4882a593Smuzhiyun 117*4882a593Smuzhiyun/* 118*4882a593Smuzhiyun * memcpy implementation 119*4882a593Smuzhiyun * 120*4882a593Smuzhiyun * void *memcpy(void *dest, const void *src, size_t n) 121*4882a593Smuzhiyun */ 122*4882a593SmuzhiyunWEAK(memcpy) 123*4882a593SmuzhiyunENTRY(__memcpy) 124*4882a593Smuzhiyun ltgr %r4,%r4 125*4882a593Smuzhiyun jz .Lmemcpy_exit 126*4882a593Smuzhiyun aghi %r4,-1 127*4882a593Smuzhiyun srlg %r5,%r4,8 128*4882a593Smuzhiyun ltgr %r5,%r5 129*4882a593Smuzhiyun lgr %r1,%r2 130*4882a593Smuzhiyun jnz .Lmemcpy_loop 131*4882a593Smuzhiyun.Lmemcpy_remainder: 132*4882a593Smuzhiyun larl %r5,.Lmemcpy_mvc 133*4882a593Smuzhiyun ex %r4,0(%r5) 134*4882a593Smuzhiyun.Lmemcpy_exit: 135*4882a593Smuzhiyun BR_EX %r14 136*4882a593Smuzhiyun.Lmemcpy_loop: 137*4882a593Smuzhiyun mvc 0(256,%r1),0(%r3) 138*4882a593Smuzhiyun la %r1,256(%r1) 139*4882a593Smuzhiyun la %r3,256(%r3) 140*4882a593Smuzhiyun brctg %r5,.Lmemcpy_loop 141*4882a593Smuzhiyun j .Lmemcpy_remainder 142*4882a593Smuzhiyun.Lmemcpy_mvc: 143*4882a593Smuzhiyun mvc 0(1,%r1),0(%r3) 144*4882a593SmuzhiyunENDPROC(__memcpy) 145*4882a593SmuzhiyunEXPORT_SYMBOL(memcpy) 146*4882a593Smuzhiyun 147*4882a593Smuzhiyun/* 148*4882a593Smuzhiyun * __memset16/32/64 149*4882a593Smuzhiyun * 150*4882a593Smuzhiyun * void *__memset16(uint16_t *s, uint16_t v, size_t count) 151*4882a593Smuzhiyun * void *__memset32(uint32_t *s, uint32_t v, size_t count) 152*4882a593Smuzhiyun * void *__memset64(uint64_t *s, uint64_t v, size_t count) 153*4882a593Smuzhiyun */ 154*4882a593Smuzhiyun.macro __MEMSET bits,bytes,insn 155*4882a593SmuzhiyunENTRY(__memset\bits) 156*4882a593Smuzhiyun ltgr %r4,%r4 157*4882a593Smuzhiyun jz .L__memset_exit\bits 158*4882a593Smuzhiyun cghi %r4,\bytes 159*4882a593Smuzhiyun je .L__memset_store\bits 160*4882a593Smuzhiyun aghi %r4,-(\bytes+1) 161*4882a593Smuzhiyun srlg %r5,%r4,8 162*4882a593Smuzhiyun ltgr %r5,%r5 163*4882a593Smuzhiyun lgr %r1,%r2 164*4882a593Smuzhiyun jz .L__memset_remainder\bits 165*4882a593Smuzhiyun.L__memset_loop\bits: 166*4882a593Smuzhiyun \insn %r3,0(%r1) 167*4882a593Smuzhiyun mvc \bytes(256-\bytes,%r1),0(%r1) 168*4882a593Smuzhiyun la %r1,256(%r1) 169*4882a593Smuzhiyun brctg %r5,.L__memset_loop\bits 170*4882a593Smuzhiyun.L__memset_remainder\bits: 171*4882a593Smuzhiyun \insn %r3,0(%r1) 172*4882a593Smuzhiyun larl %r5,.L__memset_mvc\bits 173*4882a593Smuzhiyun ex %r4,0(%r5) 174*4882a593Smuzhiyun BR_EX %r14 175*4882a593Smuzhiyun.L__memset_store\bits: 176*4882a593Smuzhiyun \insn %r3,0(%r2) 177*4882a593Smuzhiyun.L__memset_exit\bits: 178*4882a593Smuzhiyun BR_EX %r14 179*4882a593Smuzhiyun.L__memset_mvc\bits: 180*4882a593Smuzhiyun mvc \bytes(1,%r1),0(%r1) 181*4882a593SmuzhiyunENDPROC(__memset\bits) 182*4882a593Smuzhiyun.endm 183*4882a593Smuzhiyun 184*4882a593Smuzhiyun__MEMSET 16,2,sth 185*4882a593SmuzhiyunEXPORT_SYMBOL(__memset16) 186*4882a593Smuzhiyun 187*4882a593Smuzhiyun__MEMSET 32,4,st 188*4882a593SmuzhiyunEXPORT_SYMBOL(__memset32) 189*4882a593Smuzhiyun 190*4882a593Smuzhiyun__MEMSET 64,8,stg 191*4882a593SmuzhiyunEXPORT_SYMBOL(__memset64) 192