1*4882a593Smuzhiyun/* SPDX-License-Identifier: GPL-2.0-only */ 2*4882a593Smuzhiyun/* 3*4882a593Smuzhiyun * Copyright (C) 2012 ARM Ltd. 4*4882a593Smuzhiyun */ 5*4882a593Smuzhiyun 6*4882a593Smuzhiyun#include <linux/linkage.h> 7*4882a593Smuzhiyun#include <linux/const.h> 8*4882a593Smuzhiyun#include <asm/assembler.h> 9*4882a593Smuzhiyun#include <asm/page.h> 10*4882a593Smuzhiyun#include <asm/cpufeature.h> 11*4882a593Smuzhiyun#include <asm/alternative.h> 12*4882a593Smuzhiyun 13*4882a593Smuzhiyun/* 14*4882a593Smuzhiyun * Copy a page from src to dest (both are page aligned) 15*4882a593Smuzhiyun * 16*4882a593Smuzhiyun * Parameters: 17*4882a593Smuzhiyun * x0 - dest 18*4882a593Smuzhiyun * x1 - src 19*4882a593Smuzhiyun */ 20*4882a593SmuzhiyunSYM_FUNC_START_PI(copy_page) 21*4882a593Smuzhiyunalternative_if ARM64_HAS_NO_HW_PREFETCH 22*4882a593Smuzhiyun // Prefetch three cache lines ahead. 23*4882a593Smuzhiyun prfm pldl1strm, [x1, #128] 24*4882a593Smuzhiyun prfm pldl1strm, [x1, #256] 25*4882a593Smuzhiyun prfm pldl1strm, [x1, #384] 26*4882a593Smuzhiyunalternative_else_nop_endif 27*4882a593Smuzhiyun 28*4882a593Smuzhiyun ldp x2, x3, [x1] 29*4882a593Smuzhiyun ldp x4, x5, [x1, #16] 30*4882a593Smuzhiyun ldp x6, x7, [x1, #32] 31*4882a593Smuzhiyun ldp x8, x9, [x1, #48] 32*4882a593Smuzhiyun ldp x10, x11, [x1, #64] 33*4882a593Smuzhiyun ldp x12, x13, [x1, #80] 34*4882a593Smuzhiyun ldp x14, x15, [x1, #96] 35*4882a593Smuzhiyun ldp x16, x17, [x1, #112] 36*4882a593Smuzhiyun 37*4882a593Smuzhiyun add x0, x0, #256 38*4882a593Smuzhiyun add x1, x1, #128 39*4882a593Smuzhiyun1: 40*4882a593Smuzhiyun tst x0, #(PAGE_SIZE - 1) 41*4882a593Smuzhiyun 42*4882a593Smuzhiyunalternative_if ARM64_HAS_NO_HW_PREFETCH 43*4882a593Smuzhiyun prfm pldl1strm, [x1, #384] 44*4882a593Smuzhiyunalternative_else_nop_endif 45*4882a593Smuzhiyun 46*4882a593Smuzhiyun stnp x2, x3, [x0, #-256] 47*4882a593Smuzhiyun ldp x2, x3, [x1] 48*4882a593Smuzhiyun stnp x4, x5, [x0, #16 - 256] 49*4882a593Smuzhiyun ldp x4, x5, [x1, #16] 50*4882a593Smuzhiyun stnp x6, x7, [x0, #32 - 256] 51*4882a593Smuzhiyun ldp x6, x7, [x1, #32] 52*4882a593Smuzhiyun stnp x8, x9, [x0, #48 - 256] 53*4882a593Smuzhiyun ldp x8, x9, [x1, #48] 54*4882a593Smuzhiyun stnp x10, x11, [x0, #64 - 256] 55*4882a593Smuzhiyun ldp x10, x11, [x1, #64] 56*4882a593Smuzhiyun stnp x12, x13, [x0, #80 - 256] 57*4882a593Smuzhiyun ldp x12, x13, [x1, #80] 58*4882a593Smuzhiyun stnp x14, x15, [x0, #96 - 256] 59*4882a593Smuzhiyun ldp x14, x15, [x1, #96] 60*4882a593Smuzhiyun stnp x16, x17, [x0, #112 - 256] 61*4882a593Smuzhiyun ldp x16, x17, [x1, #112] 62*4882a593Smuzhiyun 63*4882a593Smuzhiyun add x0, x0, #128 64*4882a593Smuzhiyun add x1, x1, #128 65*4882a593Smuzhiyun 66*4882a593Smuzhiyun b.ne 1b 67*4882a593Smuzhiyun 68*4882a593Smuzhiyun stnp x2, x3, [x0, #-256] 69*4882a593Smuzhiyun stnp x4, x5, [x0, #16 - 256] 70*4882a593Smuzhiyun stnp x6, x7, [x0, #32 - 256] 71*4882a593Smuzhiyun stnp x8, x9, [x0, #48 - 256] 72*4882a593Smuzhiyun stnp x10, x11, [x0, #64 - 256] 73*4882a593Smuzhiyun stnp x12, x13, [x0, #80 - 256] 74*4882a593Smuzhiyun stnp x14, x15, [x0, #96 - 256] 75*4882a593Smuzhiyun stnp x16, x17, [x0, #112 - 256] 76*4882a593Smuzhiyun 77*4882a593Smuzhiyun ret 78*4882a593SmuzhiyunSYM_FUNC_END_PI(copy_page) 79*4882a593SmuzhiyunEXPORT_SYMBOL(copy_page) 80