1*4882a593Smuzhiyun/* SPDX-License-Identifier: GPL-2.0-only */ 2*4882a593Smuzhiyun/* 3*4882a593Smuzhiyun * Copyright (C) 2020 ARM Ltd. 4*4882a593Smuzhiyun */ 5*4882a593Smuzhiyun#include <linux/linkage.h> 6*4882a593Smuzhiyun 7*4882a593Smuzhiyun#include <asm/asm-uaccess.h> 8*4882a593Smuzhiyun#include <asm/assembler.h> 9*4882a593Smuzhiyun#include <asm/mte.h> 10*4882a593Smuzhiyun#include <asm/page.h> 11*4882a593Smuzhiyun#include <asm/sysreg.h> 12*4882a593Smuzhiyun 13*4882a593Smuzhiyun .arch armv8.5-a+memtag 14*4882a593Smuzhiyun 15*4882a593Smuzhiyun/* 16*4882a593Smuzhiyun * multitag_transfer_size - set \reg to the block size that is accessed by the 17*4882a593Smuzhiyun * LDGM/STGM instructions. 18*4882a593Smuzhiyun */ 19*4882a593Smuzhiyun .macro multitag_transfer_size, reg, tmp 20*4882a593Smuzhiyun mrs_s \reg, SYS_GMID_EL1 21*4882a593Smuzhiyun ubfx \reg, \reg, #SYS_GMID_EL1_BS_SHIFT, #SYS_GMID_EL1_BS_SIZE 22*4882a593Smuzhiyun mov \tmp, #4 23*4882a593Smuzhiyun lsl \reg, \tmp, \reg 24*4882a593Smuzhiyun .endm 25*4882a593Smuzhiyun 26*4882a593Smuzhiyun/* 27*4882a593Smuzhiyun * Clear the tags in a page 28*4882a593Smuzhiyun * x0 - address of the page to be cleared 29*4882a593Smuzhiyun */ 30*4882a593SmuzhiyunSYM_FUNC_START(mte_clear_page_tags) 31*4882a593Smuzhiyun multitag_transfer_size x1, x2 32*4882a593Smuzhiyun1: stgm xzr, [x0] 33*4882a593Smuzhiyun add x0, x0, x1 34*4882a593Smuzhiyun tst x0, #(PAGE_SIZE - 1) 35*4882a593Smuzhiyun b.ne 1b 36*4882a593Smuzhiyun ret 37*4882a593SmuzhiyunSYM_FUNC_END(mte_clear_page_tags) 38*4882a593Smuzhiyun 39*4882a593Smuzhiyun/* 40*4882a593Smuzhiyun * Zero the page and tags at the same time 41*4882a593Smuzhiyun * 42*4882a593Smuzhiyun * Parameters: 43*4882a593Smuzhiyun * x0 - address to the beginning of the page 44*4882a593Smuzhiyun */ 45*4882a593SmuzhiyunSYM_FUNC_START(mte_zero_clear_page_tags) 46*4882a593Smuzhiyun and x0, x0, #(1 << MTE_TAG_SHIFT) - 1 // clear the tag 47*4882a593Smuzhiyun mrs x1, dczid_el0 48*4882a593Smuzhiyun tbnz x1, #4, 2f // Branch if DC GZVA is prohibited 49*4882a593Smuzhiyun and w1, w1, #0xf 50*4882a593Smuzhiyun mov x2, #4 51*4882a593Smuzhiyun lsl x1, x2, x1 52*4882a593Smuzhiyun 53*4882a593Smuzhiyun1: dc gzva, x0 54*4882a593Smuzhiyun add x0, x0, x1 55*4882a593Smuzhiyun tst x0, #(PAGE_SIZE - 1) 56*4882a593Smuzhiyun b.ne 1b 57*4882a593Smuzhiyun ret 58*4882a593Smuzhiyun 59*4882a593Smuzhiyun2: stz2g x0, [x0], #(MTE_GRANULE_SIZE * 2) 60*4882a593Smuzhiyun tst x0, #(PAGE_SIZE - 1) 61*4882a593Smuzhiyun b.ne 2b 62*4882a593Smuzhiyun ret 63*4882a593SmuzhiyunSYM_FUNC_END(mte_zero_clear_page_tags) 64*4882a593Smuzhiyun 65*4882a593Smuzhiyun/* 66*4882a593Smuzhiyun * Copy the tags from the source page to the destination one 67*4882a593Smuzhiyun * x0 - address of the destination page 68*4882a593Smuzhiyun * x1 - address of the source page 69*4882a593Smuzhiyun */ 70*4882a593SmuzhiyunSYM_FUNC_START(mte_copy_page_tags) 71*4882a593Smuzhiyun mov x2, x0 72*4882a593Smuzhiyun mov x3, x1 73*4882a593Smuzhiyun multitag_transfer_size x5, x6 74*4882a593Smuzhiyun1: ldgm x4, [x3] 75*4882a593Smuzhiyun stgm x4, [x2] 76*4882a593Smuzhiyun add x2, x2, x5 77*4882a593Smuzhiyun add x3, x3, x5 78*4882a593Smuzhiyun tst x2, #(PAGE_SIZE - 1) 79*4882a593Smuzhiyun b.ne 1b 80*4882a593Smuzhiyun ret 81*4882a593SmuzhiyunSYM_FUNC_END(mte_copy_page_tags) 82*4882a593Smuzhiyun 83*4882a593Smuzhiyun/* 84*4882a593Smuzhiyun * Read tags from a user buffer (one tag per byte) and set the corresponding 85*4882a593Smuzhiyun * tags at the given kernel address. Used by PTRACE_POKEMTETAGS. 86*4882a593Smuzhiyun * x0 - kernel address (to) 87*4882a593Smuzhiyun * x1 - user buffer (from) 88*4882a593Smuzhiyun * x2 - number of tags/bytes (n) 89*4882a593Smuzhiyun * Returns: 90*4882a593Smuzhiyun * x0 - number of tags read/set 91*4882a593Smuzhiyun */ 92*4882a593SmuzhiyunSYM_FUNC_START(mte_copy_tags_from_user) 93*4882a593Smuzhiyun mov x3, x1 94*4882a593Smuzhiyun cbz x2, 2f 95*4882a593Smuzhiyun1: 96*4882a593Smuzhiyun uao_user_alternative 2f, ldrb, ldtrb, w4, x1, 0 97*4882a593Smuzhiyun lsl x4, x4, #MTE_TAG_SHIFT 98*4882a593Smuzhiyun stg x4, [x0], #MTE_GRANULE_SIZE 99*4882a593Smuzhiyun add x1, x1, #1 100*4882a593Smuzhiyun subs x2, x2, #1 101*4882a593Smuzhiyun b.ne 1b 102*4882a593Smuzhiyun 103*4882a593Smuzhiyun // exception handling and function return 104*4882a593Smuzhiyun2: sub x0, x1, x3 // update the number of tags set 105*4882a593Smuzhiyun ret 106*4882a593SmuzhiyunSYM_FUNC_END(mte_copy_tags_from_user) 107*4882a593Smuzhiyun 108*4882a593Smuzhiyun/* 109*4882a593Smuzhiyun * Get the tags from a kernel address range and write the tag values to the 110*4882a593Smuzhiyun * given user buffer (one tag per byte). Used by PTRACE_PEEKMTETAGS. 111*4882a593Smuzhiyun * x0 - user buffer (to) 112*4882a593Smuzhiyun * x1 - kernel address (from) 113*4882a593Smuzhiyun * x2 - number of tags/bytes (n) 114*4882a593Smuzhiyun * Returns: 115*4882a593Smuzhiyun * x0 - number of tags read/set 116*4882a593Smuzhiyun */ 117*4882a593SmuzhiyunSYM_FUNC_START(mte_copy_tags_to_user) 118*4882a593Smuzhiyun mov x3, x0 119*4882a593Smuzhiyun cbz x2, 2f 120*4882a593Smuzhiyun1: 121*4882a593Smuzhiyun ldg x4, [x1] 122*4882a593Smuzhiyun ubfx x4, x4, #MTE_TAG_SHIFT, #MTE_TAG_SIZE 123*4882a593Smuzhiyun uao_user_alternative 2f, strb, sttrb, w4, x0, 0 124*4882a593Smuzhiyun add x0, x0, #1 125*4882a593Smuzhiyun add x1, x1, #MTE_GRANULE_SIZE 126*4882a593Smuzhiyun subs x2, x2, #1 127*4882a593Smuzhiyun b.ne 1b 128*4882a593Smuzhiyun 129*4882a593Smuzhiyun // exception handling and function return 130*4882a593Smuzhiyun2: sub x0, x0, x3 // update the number of tags copied 131*4882a593Smuzhiyun ret 132*4882a593SmuzhiyunSYM_FUNC_END(mte_copy_tags_to_user) 133*4882a593Smuzhiyun 134*4882a593Smuzhiyun/* 135*4882a593Smuzhiyun * Save the tags in a page 136*4882a593Smuzhiyun * x0 - page address 137*4882a593Smuzhiyun * x1 - tag storage 138*4882a593Smuzhiyun */ 139*4882a593SmuzhiyunSYM_FUNC_START(mte_save_page_tags) 140*4882a593Smuzhiyun multitag_transfer_size x7, x5 141*4882a593Smuzhiyun1: 142*4882a593Smuzhiyun mov x2, #0 143*4882a593Smuzhiyun2: 144*4882a593Smuzhiyun ldgm x5, [x0] 145*4882a593Smuzhiyun orr x2, x2, x5 146*4882a593Smuzhiyun add x0, x0, x7 147*4882a593Smuzhiyun tst x0, #0xFF // 16 tag values fit in a register, 148*4882a593Smuzhiyun b.ne 2b // which is 16*16=256 bytes 149*4882a593Smuzhiyun 150*4882a593Smuzhiyun str x2, [x1], #8 151*4882a593Smuzhiyun 152*4882a593Smuzhiyun tst x0, #(PAGE_SIZE - 1) 153*4882a593Smuzhiyun b.ne 1b 154*4882a593Smuzhiyun 155*4882a593Smuzhiyun ret 156*4882a593SmuzhiyunSYM_FUNC_END(mte_save_page_tags) 157*4882a593Smuzhiyun 158*4882a593Smuzhiyun/* 159*4882a593Smuzhiyun * Restore the tags in a page 160*4882a593Smuzhiyun * x0 - page address 161*4882a593Smuzhiyun * x1 - tag storage 162*4882a593Smuzhiyun */ 163*4882a593SmuzhiyunSYM_FUNC_START(mte_restore_page_tags) 164*4882a593Smuzhiyun multitag_transfer_size x7, x5 165*4882a593Smuzhiyun1: 166*4882a593Smuzhiyun ldr x2, [x1], #8 167*4882a593Smuzhiyun2: 168*4882a593Smuzhiyun stgm x2, [x0] 169*4882a593Smuzhiyun add x0, x0, x7 170*4882a593Smuzhiyun tst x0, #0xFF 171*4882a593Smuzhiyun b.ne 2b 172*4882a593Smuzhiyun 173*4882a593Smuzhiyun tst x0, #(PAGE_SIZE - 1) 174*4882a593Smuzhiyun b.ne 1b 175*4882a593Smuzhiyun 176*4882a593Smuzhiyun ret 177*4882a593SmuzhiyunSYM_FUNC_END(mte_restore_page_tags) 178